A desktop coding agent with free access to frontier models.
TypeScript
feat(app): voice dictation in the composer and Settings → Voice
Samrat Talukdar692345845608
diff --git a/apps/app/src/app/lib/omnirush-server.ts b/apps/app/src/app/lib/omnirush-server.tsindex 389bb222..48473eb0 100644--- a/apps/app/src/app/lib/omnirush-server.ts+++ b/apps/app/src/app/lib/omnirush-server.ts@@ -407,6 +407,20 @@ export type OmniRushSubagentModelSetting = {effort: string | null;};+/** GET /omnirush/voice/status: whether dictation can be used, and hint words for the project. */+export type OmniRushVoiceStatus = {+ signedIn: boolean;+ /** Null: the account's catalog does not say; the first upload decides. */+ available: boolean | null;+ reason: string | null;+ /** Days the speech provider keeps uploaded audio, when the backend says (for the privacy note). */+ providerRetentionDays?: number | null;+ /** Seconds of dictation left today, when the account has a daily cap. */+ remainingSecondsToday?: number | null;+ repo: string | null;+ branch: string | null;+};+export type OmniRushSubagentModelState = {setting: OmniRushSubagentModelSetting;/** Whether an omnirush.ai account is signed in (its models can run). */@@ -1859,6 +1873,17 @@ export function createOmniRushServerClient(options: { baseUrl: string; token?: sbody: { mode },timeoutMs: timeouts.config,}),+ getVoiceStatus: (workspaceId?: string | null) =>+ requestJson<OmniRushVoiceStatus>(+ baseUrl,+ `/omnirush/voice/status${workspaceId ? `?workspace=${encodeURIComponent(workspaceId)}` : ""}`,+ { token, hostToken, timeoutMs: timeouts.config },+ ),+ /** Where the renderer POSTs one voice segment (multipart), and its headers; the local server adds the device bearer. */+ voiceTranscriptionEndpoint: () => ({+ url: `${baseUrl}/omnirush/voice/transcribe`,+ headers: buildAuthHeaders(token, hostToken),+ }),getSubagentModel: () =>requestJson<OmniRushSubagentModelState>(baseUrl, "/omnirush/subagent-model", { token, hostToken, timeoutMs: timeouts.config }),setSubagentModel: (setting: OmniRushSubagentModelSetting) =>diff --git a/apps/app/src/app/types.ts b/apps/app/src/app/types.tsindex 709dbb9b..7a4eb6d5 100644--- a/apps/app/src/app/types.ts+++ b/apps/app/src/app/types.ts@@ -199,6 +199,7 @@ export const SETTINGS_TAB_VALUES = ["environment","advanced","appearance",+ "voice","updates","recovery","debug",diff --git a/apps/app/src/components/model-select.tsx b/apps/app/src/components/model-select.tsxindex 3a10de17..cbc38031 100644--- a/apps/app/src/components/model-select.tsx+++ b/apps/app/src/components/model-select.tsx@@ -453,7 +453,7 @@ export function ModelSelect({disabled={disabled}aria-label="Change model"aria-keyshortcuts="Meta+Alt+/"- className="flex h-9 max-h-9 min-w-0 max-w-fit flex-1 items-center gap-1.5 rounded-md px-2.5 text-sm text-gray-10 transition-colors hover:bg-gray-3 hover:text-gray-12 disabled:pointer-events-none disabled:opacity-60"+ className="flex h-9 max-h-9 min-w-0 max-w-fit flex-1 items-center overflow-hidden gap-1.5 rounded-md px-2.5 text-sm text-gray-10 transition-colors hover:bg-gray-3 hover:text-gray-12 disabled:pointer-events-none disabled:opacity-60"/>}>diff --git a/apps/app/src/react-app/domains/session/chat/new-task-composer.tsx b/apps/app/src/react-app/domains/session/chat/new-task-composer.tsxindex 9af63da3..2ef4fb2f 100644--- a/apps/app/src/react-app/domains/session/chat/new-task-composer.tsx+++ b/apps/app/src/react-app/domains/session/chat/new-task-composer.tsx@@ -427,6 +427,8 @@ export function NewTaskComposer(props: NewTaskComposerProps) {runModeControl={<WorkspaceRunModeMenu client={workspaceClient} workspaceId={workspaceId} busy={props.busy} />}fullPermissionsControl={<FullPermissionsToggle client={workspaceClient} workspaceId={workspaceId} />}subagentModelControl={<SubagentModelMenu client={workspaceClient} />}+ voiceClient={workspaceClient}+ voiceWorkspaceId={workspaceId}draft={props.draft}mentions={mentions}onDraftChange={handleDraftChange}diff --git a/apps/app/src/react-app/domains/session/surface/composer/composer.tsx b/apps/app/src/react-app/domains/session/surface/composer/composer.tsxindex c95dd464..e85fa44d 100644--- a/apps/app/src/react-app/domains/session/surface/composer/composer.tsx+++ b/apps/app/src/react-app/domains/session/surface/composer/composer.tsx@@ -36,6 +36,8 @@ import {mergeComposerConnectionInventory,} from "./composer-connections";import { DevProfiler } from "@/react-app/shell/dev-profiler";+import { useVoiceDictation, type VoiceClient } from "@/react-app/domains/voice/use-voice-dictation";+import { VoiceButton, VoicePermissionCard } from "@/react-app/domains/voice/voice-button";type MentionItem = {id: string;@@ -122,6 +124,10 @@ type ComposerProps = {fullPermissionsControl?: ReactNode;/** The "Sub-agents" model and effort menu, shown right after the model selector. */subagentModelControl?: ReactNode;+ /** Voice dictation's local server hop; without it there is no mic button. */+ voiceClient?: VoiceClient | null;+ /** The workspace whose repository and branch hint the transcription. */+ voiceWorkspaceId?: string | null;};const FLUSH_PROMPT_EVENT = "omnirush:flushPromptDraft";@@ -352,7 +358,24 @@ export const ReactSessionComposer = memo(function ReactSessionComposer(props: Co// Editor submit (Enter). While idle this sends normally; while busy// Enter queues until the agent finishes, and Cmd/Ctrl+Enter steers.+ // Auto-send after dictation waits until the dictated text has reached the draft.+ const [voiceAutoSendPrompt, setVoiceAutoSendPrompt] = useState<string | null>(null);+ const voice = useVoiceDictation({+ client: props.voiceClient ?? null,+ workspaceId: props.voiceWorkspaceId,+ recentFiles: props.recentFiles,+ editorRef,+ rootRef,+ disabled: props.disabled,+ onAutoSend: setVoiceAutoSendPrompt,+ });+const handleEditorSubmit = useCallback((options: { queue: boolean }) => {+ // Enter while dictating ends the dictation; the text is then editable before sending.+ if (voice.active) {+ void voice.stop();+ return;+ }const hasContent = props.draft.trim().length > 0 || props.attachments.length > 0;if (!hasContent) return;if (props.submissionPreparing) return;@@ -362,7 +385,13 @@ export const ReactSessionComposer = memo(function ReactSessionComposer(props: Coreturn;}void props.onSend();- }, [props.busy, props.draft, props.attachments, props.onSend, props.onSteer, props.onQueue, props.submissionPreparing]);+ }, [props.busy, props.draft, props.attachments, props.onSend, props.onSteer, props.onQueue, props.submissionPreparing, voice.active, voice.stop]);++ useEffect(() => {+ if (voiceAutoSendPrompt === null || props.draft !== voiceAutoSendPrompt) return;+ setVoiceAutoSendPrompt(null);+ handleEditorSubmit({ queue: false });+ }, [handleEditorSubmit, props.draft, voiceAutoSendPrompt]);const slashCommandQuery = getSlashCommandQuery(props.draft);const slashOpenNext = slashCommandQuery !== null;@@ -1287,6 +1316,7 @@ export const ReactSessionComposer = memo(function ReactSessionComposer(props: CoclassName={props.flush ? `relative ${toolMenuOpen ? "z-50" : "z-20"}` : `sticky bottom-0 ${toolMenuOpen ? "z-50" : "z-20"} bg-gradient-to-t from-dls-surface via-dls-surface/95 to-transparent px-4 pb-[max(0.5rem,calc(env(safe-area-inset-bottom)+var(--keyboard-inset,0px)))] max-lg:px-3 lg:px-8 ${props.compactTopSpacing ? "pt-0" : "pt-1"}`}style={{ contain: "layout style" }}onKeyDownCapture={handleKeyDownCapture}+ onFocusCapture={voice.claimHotkey}onCompositionStart={() => {imeComposingRef.current = true;}}@@ -1300,6 +1330,7 @@ export const ReactSessionComposer = memo(function ReactSessionComposer(props: CoclassName={`@container/composer relative overflow-visible rounded-[18px] border border-dls-border bg-dls-surface transition-all ${panelRoundedClass}`}>{props.topAccessory ? <div className="relative z-10">{props.topAccessory}</div> : null}+ <VoicePermissionCard voice={voice} />{renderMentionMenu()}{renderSlashMenu()}@@ -1675,6 +1706,7 @@ export const ReactSessionComposer = memo(function ReactSessionComposer(props: Co><Paperclip size={16} /></button>+ <VoiceButton voice={voice} disabled={props.disabled} /></div>@@ -1815,7 +1847,7 @@ export const ReactSessionComposer = memo(function ReactSessionComposer(props: Co}disabled={props.disabled- || (!props.busy && (!canSend || props.submissionPreparing))+ || (!props.busy && (!canSend || props.submissionPreparing || voice.active))}aria-label={props.busydiff --git a/apps/app/src/react-app/domains/session/surface/composer/editor.tsx b/apps/app/src/react-app/domains/session/surface/composer/editor.tsxindex 10788817..2771fc5b 100644--- a/apps/app/src/react-app/domains/session/surface/composer/editor.tsx+++ b/apps/app/src/react-app/domains/session/surface/composer/editor.tsx@@ -38,6 +38,7 @@ import { decodeComposerMentionValue, encodeComposerMentionValue, type ComposerMeimport { parseConnectSkillToken } from "./connect-skill-token";import { shouldCollapsePastedText, splitPastedText } from "./pasted-text";import { insertPastedText } from "./pasted-text-insertion";+import { padInsertion } from "@/app/lib/voice-core/text";type PastedTextToken = { label: string; lines: number; text: string };@@ -68,6 +69,14 @@ type EditorProps = {export type LexicalPromptEditorHandle = {insertSkillAtSelection: (skillName: string, skillToken?: string) => void;+ /** Voice dictation: marks the cursor with a live (dim) transcript placeholder. */+ voiceBegin: () => void;+ /** The dictated text so far, and whether more is still being transcribed. False once the placeholder is gone. */+ voiceUpdate: (text: string, pending: boolean) => boolean;+ /** Replaces the placeholder with plain, editable text and puts the cursor after it; returns the whole prompt. */+ voiceCommit: (text: string) => string;+ /** Removes the placeholder: the prompt is as it was before dictation. */+ voiceCancel: () => void;};type SerializedComposerMentionNode = Spread<@@ -661,6 +670,132 @@ type ComposerInlineTokenNode =| ComposerPastedTextNode| ComposerAttachmentNode;+const VOICE_EMPTY = "\u200b";+const VOICE_NODE_CLASS = "rounded-sm bg-[color:color-mix(in_oklab,var(--dls-accent)_10%,transparent)] italic text-dls-secondary";+const VOICE_PENDING_CLASS = "after:ml-0.5 after:inline-block after:animate-pulse after:content-['…']";++/**+ * The live transcript while dictating: dim, italic, not editable, at the+ * cursor where recording started. Committing turns it into a plain text+ * node; cancelling removes it.+ */+class ComposerVoiceNode extends TextNode {+ __pending: boolean;++ static override getType() {+ return "composer-voice";+ }++ static override clone(node: ComposerVoiceNode) {+ const clone = new ComposerVoiceNode(node.__text, node.__key);+ clone.__pending = node.__pending;+ return clone;+ }++ static override importJSON(serializedNode: SerializedTextNode) {+ return $applyNodeReplacement(new ComposerVoiceNode(serializedNode.text));+ }++ constructor(text = VOICE_EMPTY, key?: NodeKey) {+ super(text || VOICE_EMPTY, key);+ this.__pending = true;+ }++ override exportJSON(): SerializedTextNode {+ return { ...super.exportJSON(), type: "composer-voice", version: 1 };+ }++ override getTextContent(): string {+ return super.getTextContent().replaceAll(VOICE_EMPTY, "");+ }++ override createDOM(config: EditorConfig) {+ const dom = super.createDOM(config);+ dom.className = `${VOICE_NODE_CLASS} ${this.__pending ? VOICE_PENDING_CLASS : ""}`;+ dom.contentEditable = "false";+ dom.setAttribute("data-voice-interim", "true");+ dom.setAttribute("data-voice-pending", String(this.__pending));+ return dom;+ }++ override updateDOM(prevNode: this, dom: HTMLElement, config: EditorConfig) {+ const changed = super.updateDOM(prevNode, dom, config);+ if (prevNode.__pending !== this.__pending) {+ dom.className = `${VOICE_NODE_CLASS} ${this.__pending ? VOICE_PENDING_CLASS : ""}`;+ dom.setAttribute("data-voice-pending", String(this.__pending));+ }+ return changed;+ }++ setVoice(text: string, pending: boolean) {+ const writable = this.getWritable();+ writable.__text = text || VOICE_EMPTY;+ writable.__pending = pending;+ }++ override canInsertTextBefore(): false {+ return false;+ }++ override canInsertTextAfter(): false {+ return false;+ }++ override isToken(): true {+ return true;+ }+}++function neighbourText(node: TextNode, side: "before" | "after"): string {+ const sibling = side === "before" ? node.getPreviousSibling() : node.getNextSibling();+ return sibling ? sibling.getTextContent() : "";+}++function voiceNode(): ComposerVoiceNode | null {+ return $nodesOfType(ComposerVoiceNode)[0] ?? null;+}++function beginVoice() {+ for (const stale of $nodesOfType(ComposerVoiceNode)) stale.remove();+ const node = $applyNodeReplacement(new ComposerVoiceNode());+ const selection = $getSelection();+ if ($isRangeSelection(selection)) {+ if (!selection.isCollapsed()) selection.removeText();+ selection.insertNodes([node]);+ } else {+ const root = $getRoot();+ const last = root.getLastChild();+ const paragraph = $isElementNode(last) ? last : $createParagraphNode();+ if (!$isElementNode(last)) root.append(paragraph);+ paragraph.append(node);+ }+}++function updateVoice(text: string, pending: boolean): boolean {+ const node = voiceNode();+ if (!node) return false;+ node.setVoice(padInsertion(neighbourText(node, "before"), text, neighbourText(node, "after")), pending);+ return true;+}++function commitVoice(text: string) {+ const node = voiceNode();+ if (!node) {+ // The placeholder was removed (a draft rebuild): the words still land at the cursor.+ const selection = $getSelection();+ if (text.trim() && $isRangeSelection(selection)) selection.insertText(text.trim());+ return;+ }+ const padded = padInsertion(neighbourText(node, "before"), text, neighbourText(node, "after"));+ if (!padded) {+ node.remove();+ return;+ }+ const plain = $createTextNode(padded);+ node.replace(plain);+ plain.select(padded.length, padded.length);+}+function isComposerInlineTokenNode(node: unknown): node is ComposerInlineTokenNode {return node instanceof ComposerMentionNode|| node instanceof ComposerSlashCommandNode@@ -1166,6 +1301,28 @@ function ImperativeHandlePlugin(props: { editorRef: ForwardedRef<LexicalPromptEdeditor.update(() => insertSkillAtSelection(skillName, skillToken));editor.focus();},+ voiceBegin() {+ editor.update(() => beginVoice(), { discrete: true });+ // Started from the hotkey anywhere in the window: the composer takes focus.+ editor.focus();+ },+ voiceUpdate(text: string, pending: boolean) {+ let present = false;+ editor.update(() => {+ present = updateVoice(text, pending);+ }, { discrete: true, tag: "history-merge" });+ return present;+ },+ voiceCommit(text: string) {+ editor.update(() => commitVoice(text), { discrete: true });+ editor.focus();+ return editor.getEditorState().read(() => serializePromptFromRoot());+ },+ voiceCancel() {+ editor.update(() => {+ for (const node of $nodesOfType(ComposerVoiceNode)) node.remove();+ }, { discrete: true, tag: "history-merge" });+ },}), [editor]);return null;@@ -1232,7 +1389,7 @@ export const LexicalPromptEditor = forwardRef<LexicalPromptEditorHandle, EditorPthrow error;},editable: true,- nodes: [ComposerMentionNode, ComposerSlashCommandNode, ComposerSkillNode, ComposerPastedTextNode, ComposerAttachmentNode],+ nodes: [ComposerMentionNode, ComposerSlashCommandNode, ComposerSkillNode, ComposerPastedTextNode, ComposerAttachmentNode, ComposerVoiceNode],editorState: () => {setPrompt(props.value, props.mentions, props.pastedText, props.attachments);},diff --git a/apps/app/src/react-app/domains/session/surface/session-surface.tsx b/apps/app/src/react-app/domains/session/surface/session-surface.tsxindex 9953dd5e..4e945ffb 100644--- a/apps/app/src/react-app/domains/session/surface/session-surface.tsx+++ b/apps/app/src/react-app/domains/session/surface/session-surface.tsx@@ -3226,6 +3226,8 @@ export function SessionSurface(props: SessionSurfaceProps) {runModeControl={<WorkspaceRunModeMenu client={props.client} workspaceId={props.workspaceId} busy={chatStreaming || preparingCloudTools || Boolean(props.activePermission || props.activeQuestion)} />}fullPermissionsControl={<FullPermissionsToggle client={props.client} workspaceId={props.workspaceId} />}subagentModelControl={<SubagentModelMenu client={props.client} />}+ voiceClient={props.client}+ voiceWorkspaceId={props.workspaceId}draft={autoSendPayload ? draft : autoSending ? "" : draft}mentions={mentions}onDraftChange={handleComposerDraftChange}diff --git a/apps/app/src/react-app/domains/settings/pages/general-view.tsx b/apps/app/src/react-app/domains/settings/pages/general-view.tsxindex 3c6de53d..24c5651c 100644--- a/apps/app/src/react-app/domains/settings/pages/general-view.tsx+++ b/apps/app/src/react-app/domains/settings/pages/general-view.tsx@@ -4,6 +4,7 @@ import {ArrowRight,Cog,FolderLock,+ Mic,Paintbrush,RefreshCcw,ShieldCheck,@@ -60,6 +61,7 @@ const workspaceCards: SettingsCardDefinition[] = [const globalCards: SettingsCardDefinition[] = [{ tab: "ai", icon: Sparkles, title: "AI Providers", desc: "Connect services that provide AI models." },{ tab: "appearance", icon: Paintbrush, title: "Appearance", desc: "Theme, font size, and display." },+ { tab: "voice", icon: Mic, title: "Voice", desc: "Dictation, microphone, and hotkey." },{ tab: "environment", icon: Terminal, title: "Environment", desc: "Environment variables and paths." },{ tab: "updates", icon: RefreshCcw, title: "Updates", desc: "App version and update channel." },];diff --git a/apps/app/src/react-app/domains/settings/shell/settings-page.tsx b/apps/app/src/react-app/domains/settings/shell/settings-page.tsxindex 4823a057..67ee1f13 100644--- a/apps/app/src/react-app/domains/settings/shell/settings-page.tsx+++ b/apps/app/src/react-app/domains/settings/shell/settings-page.tsx@@ -8,6 +8,7 @@ import {CloudCog,Cog,FolderLock,+ Mic,Paintbrush,Puzzle,RefreshCcw,@@ -86,6 +87,8 @@ export function getSettingsTabIcon(tab: SettingsTab) {return Wrench;case "appearance":return Paintbrush;+ case "voice":+ return Mic;case "updates":return RefreshCcw;case "recovery":@@ -123,6 +126,8 @@ export function getSettingsTabLabel(tab: SettingsTab) {return t("settings.tab_advanced");case "appearance":return t("settings.tab_appearance");+ case "voice":+ return "Voice";case "updates":return t("settings.tab_updates");case "recovery":@@ -162,6 +167,8 @@ export function getSettingsTabDescription(tab: SettingsTab) {return t("settings.tab_description_advanced");case "appearance":return t("settings.tab_description_appearance");+ case "voice":+ return "Dictation, microphone, and hotkey";case "updates":return t("settings.tab_description_updates");case "recovery":@@ -183,7 +190,7 @@ export function getGlobalSettingsTabs(developerMode: boolean,capabilities: Pick<PlatformCapabilities, "autoUpdate">,): SettingsTab[] {- const tabs: SettingsTab[] = ["ai", "appearance", "environment"];+ const tabs: SettingsTab[] = ["ai", "appearance", "voice", "environment"];if (capabilities.autoUpdate) tabs.push("updates");if (developerMode) tabs.push("debug");return tabs;diff --git a/apps/app/src/react-app/domains/voice/hotkey.ts b/apps/app/src/react-app/domains/voice/hotkey.tsnew file mode 100644index 00000000..863e322a--- /dev/null+++ b/apps/app/src/react-app/domains/voice/hotkey.ts@@ -0,0 +1,91 @@+/**+ * The dictation hotkey: modifiers plus a KeyboardEvent.code ("Ctrl+Space",+ * "Alt+Space", "Ctrl+Shift+KeyM"). Codes are layout-independent, and macOS+ * Option+Space still reports code "Space" even though it types a+ * non-breaking space.+ */+export type Hotkey = { ctrl: boolean; alt: boolean; shift: boolean; meta: boolean; code: string };++const MODIFIER_CODES = new Set(["ControlLeft", "ControlRight", "AltLeft", "AltRight", "ShiftLeft", "ShiftRight", "MetaLeft", "MetaRight"]);++export function isMacPlatform(): boolean {+ return typeof navigator !== "undefined" && /Macintosh|Mac OS X|iPhone|iPad/.test(navigator.userAgent);+}++/** Option+Space on macOS (Cmd+Space is Spotlight, Ctrl+Space switches input source); Ctrl+Space elsewhere (Alt+Space opens the window menu on Windows). */+export function defaultHotkey(mac = isMacPlatform()): string {+ return mac ? "Alt+Space" : "Ctrl+Space";+}++export function parseHotkey(value: string): Hotkey | null {+ const parts = value.split("+").map((part) => part.trim()).filter(Boolean);+ const code = parts.pop();+ if (!code || MODIFIER_CODES.has(code)) return null;+ const hotkey: Hotkey = { ctrl: false, alt: false, shift: false, meta: false, code };+ for (const part of parts) {+ const name = part.toLowerCase();+ if (name === "ctrl" || name === "control") hotkey.ctrl = true;+ else if (name === "alt" || name === "option") hotkey.alt = true;+ else if (name === "shift") hotkey.shift = true;+ else if (name === "meta" || name === "cmd" || name === "command" || name === "super") hotkey.meta = true;+ else return null;+ }+ // A bare letter or digit would fire while typing.+ if (!hotkey.ctrl && !hotkey.alt && !hotkey.meta && !/^F\d{1,2}$/.test(code)) return null;+ return hotkey;+}++export function hotkeyFromEvent(event: Pick<KeyboardEvent, "ctrlKey" | "altKey" | "shiftKey" | "metaKey" | "code">): string | null {+ if (MODIFIER_CODES.has(event.code) || !event.code) return null;+ const parts = [event.ctrlKey && "Ctrl", event.altKey && "Alt", event.shiftKey && "Shift", event.metaKey && "Meta", event.code].filter(+ (part): part is string => typeof part === "string",+ );+ const value = parts.join("+");+ return parseHotkey(value) ? value : null;+}++export function matchesHotkeyDown(event: Pick<KeyboardEvent, "ctrlKey" | "altKey" | "shiftKey" | "metaKey" | "code">, hotkey: Hotkey): boolean {+ return event.code === hotkey.code+ && event.ctrlKey === hotkey.ctrl+ && event.altKey === hotkey.alt+ && event.shiftKey === hotkey.shift+ && event.metaKey === hotkey.meta;+}++/** Releasing the hotkey's key ends a hold, whichever modifier was let go first. */+export function matchesHotkeyUp(event: Pick<KeyboardEvent, "code">, hotkey: Hotkey): boolean {+ return event.code === hotkey.code;+}++export function formatHotkey(value: string, mac = isMacPlatform()): string {+ const hotkey = parseHotkey(value);+ if (!hotkey) return value;+ const key = hotkey.code.replace(/^Key/, "").replace(/^Digit/, "");+ const names = mac+ ? [hotkey.ctrl && "⌃", hotkey.alt && "⌥", hotkey.shift && "⇧", hotkey.meta && "⌘"]+ : [hotkey.ctrl && "Ctrl", hotkey.alt && "Alt", hotkey.shift && "Shift", hotkey.meta && "Win"];+ const modifiers = names.filter((part): part is string => typeof part === "string");+ return mac ? `${modifiers.join("")}${key}` : [...modifiers, key].join("+");+}++/** A press shorter than this is a tap (toggle); longer is push-to-talk. */+export const HOLD_THRESHOLD_MS = 250;++export type HotkeyAction = "start" | "stop" | "none";++/**+ * The hold/tap decision, kept pure for tests. `recording` is whether a+ * dictation is running; `pressedAt` the time of the keydown that started it+ * (null when it was started by a tap or the mic button).+ */+export function hotkeyDown(mode: "both" | "hold" | "tap", recording: boolean): HotkeyAction {+ if (!recording) return "start";+ return mode === "hold" ? "none" : "stop";+}++export function hotkeyUp(mode: "both" | "hold" | "tap", recording: boolean, heldMs: number | null): HotkeyAction {+ if (!recording || heldMs === null) return "none";+ if (mode === "tap") return "none";+ if (mode === "hold") return "stop";+ return heldMs >= HOLD_THRESHOLD_MS ? "stop" : "none";+}diff --git a/apps/app/src/react-app/domains/voice/mic-source.ts b/apps/app/src/react-app/domains/voice/mic-source.tsnew file mode 100644index 00000000..b39c02e3--- /dev/null+++ b/apps/app/src/react-app/domains/voice/mic-source.ts@@ -0,0 +1,221 @@+import { Downsampler, VOICE_SAMPLE_RATE, VoiceError, type AudioSource } from "@/app/lib/voice-core";++/**+ * Posts each 128-frame render quantum of channel 0 to the main thread. The+ * worklet only forwards: resampling and VAD run in voice-core.+ */+const WORKLET_SOURCE = `+class OmniRushCaptureProcessor extends AudioWorkletProcessor {+ process(inputs) {+ const channel = inputs[0] && inputs[0][0];+ if (channel && channel.length) this.port.postMessage(channel.slice(0));+ return true;+ }+}+registerProcessor("omnirush-capture", OmniRushCaptureProcessor);+`;++/** How long the microphone stays open after a recording, so the next one starts without a cold start. */+const KEEP_WARM_MS = 30_000;++type OpenMic = {+ deviceId: string | null;+ stream: MediaStream;+ context: AudioContext;+ source: MediaStreamAudioSourceNode;+ node: AudioWorkletNode | ScriptProcessorNode;+ /** Where the worklet's samples go while a recording listens. */+ sink: { listener: ((samples: Float32Array) => void) | null };+ releaseTimer: ReturnType<typeof setTimeout> | null;+};++let open: OpenMic | null = null;+let opening: Promise<OpenMic> | null = null;++function mapCaptureError(error: unknown): VoiceError {+ const name = error instanceof DOMException || error instanceof Error ? error.name : "";+ if (name === "NotAllowedError" || name === "SecurityError" || name === "PermissionDeniedError") {+ return new VoiceError("permission_denied", "Microphone access is turned off for OmniRush.ai.");+ }+ if (name === "NotFoundError" || name === "OverconstrainedError" || name === "DevicesNotFoundError") {+ return new VoiceError("no_device", "No microphone was found. Connect one, or pick another input in Settings → Voice.");+ }+ if (name === "NotReadableError" || name === "TrackStartError" || name === "AbortError") {+ return new VoiceError("capture_failed", "The microphone is busy or could not be started. Close other apps using it and try again.");+ }+ return new VoiceError("capture_failed", "The microphone could not be opened.");+}++function release(mic: OpenMic): void {+ if (mic.releaseTimer) clearTimeout(mic.releaseTimer);+ try {+ mic.node.disconnect();+ mic.source.disconnect();+ } catch {+ // already disconnected+ }+ for (const track of mic.stream.getTracks()) track.stop();+ void mic.context.close().catch(() => undefined);+ if (open === mic) open = null;+}++/** Closes the microphone now (settings changed, app hidden). */+export function releaseMicrophone(): void {+ if (open) release(open);+}++async function openMicrophone(deviceId: string | null): Promise<OpenMic> {+ if (open && open.deviceId === deviceId && open.stream.getAudioTracks().some((track) => track.readyState === "live")) {+ if (open.releaseTimer) clearTimeout(open.releaseTimer);+ open.releaseTimer = null;+ return open;+ }+ if (open) release(open);+ if (!navigator.mediaDevices?.getUserMedia) {+ throw new VoiceError("no_device", "This window cannot use a microphone.");+ }+ let stream: MediaStream;+ try {+ stream = await navigator.mediaDevices.getUserMedia({+ audio: {+ ...(deviceId ? { deviceId: { exact: deviceId } } : {}),+ channelCount: 1,+ echoCancellation: true,+ noiseSuppression: true,+ autoGainControl: true,+ },+ video: false,+ });+ } catch (error) {+ throw mapCaptureError(error);+ }+ // Chromium resamples the input to the context rate; 16 kHz directly when it can.+ let context: AudioContext;+ try {+ context = new AudioContext({ sampleRate: VOICE_SAMPLE_RATE });+ } catch {+ context = new AudioContext();+ }+ const source = context.createMediaStreamSource(stream);+ const sink: OpenMic["sink"] = { listener: null };+ const deliver = (samples: Float32Array) => sink.listener?.(samples);+ let node: AudioWorkletNode | ScriptProcessorNode;+ try {+ const url = URL.createObjectURL(new Blob([WORKLET_SOURCE], { type: "text/javascript" }));+ try {+ await context.audioWorklet.addModule(url);+ } finally {+ URL.revokeObjectURL(url);+ }+ const worklet = new AudioWorkletNode(context, "omnirush-capture", { numberOfInputs: 1, numberOfOutputs: 0, channelCount: 1 });+ worklet.port.onmessage = (event: MessageEvent<Float32Array>) => deliver(event.data);+ node = worklet;+ } catch {+ // No AudioWorklet (very old runtime): the deprecated processor still works.+ const processor = context.createScriptProcessor(2048, 1, 1);+ processor.onaudioprocess = (event) => deliver(event.inputBuffer.getChannelData(0).slice(0));+ node = processor;+ }+ const mic: OpenMic = { deviceId, stream, context, source, node, sink, releaseTimer: null };+ source.connect(node);+ if (node instanceof ScriptProcessorNode) node.connect(context.destination);+ if (context.state === "suspended") await context.resume().catch(() => undefined);+ for (const track of stream.getAudioTracks()) {+ track.addEventListener("ended", () => {+ if (open === mic) release(mic);+ });+ }+ open = mic;+ return mic;+}++/**+ * The microphone as a voice-core AudioSource. PCM goes from the worklet to+ * voice-core in memory only; stopping keeps the device warm for 30 s, then+ * releases it (the OS indicator turns off).+ */+export class MicrophoneSource implements AudioSource {+ private mic: OpenMic | null = null;+ private stopped = false;+ private onEnd: ((error?: VoiceError) => void) | null = null;++ constructor(private readonly deviceId: string | null) {}++ async start(onFrames: (frames: Int16Array) => void, onEnd?: (error?: VoiceError) => void): Promise<void> {+ this.stopped = false;+ this.onEnd = onEnd ?? null;+ opening ??= openMicrophone(this.deviceId).finally(() => {+ opening = null;+ });+ const mic = await opening;+ if (this.stopped) {+ this.scheduleRelease(mic);+ return;+ }+ this.mic = mic;+ const downsampler = new Downsampler(mic.context.sampleRate);+ mic.sink.listener = (samples) => {+ if (!this.stopped) onFrames(downsampler.push(samples));+ };+ const track = mic.stream.getAudioTracks()[0];+ track?.addEventListener("ended", () => {+ if (!this.stopped) this.onEnd?.(new VoiceError("no_device", "The microphone was disconnected."));+ }, { once: true });+ }++ stop(): void {+ this.stopped = true;+ const mic = this.mic;+ this.mic = null;+ if (!mic) return;+ mic.sink.listener = null;+ this.scheduleRelease(mic);+ }++ private scheduleRelease(mic: OpenMic): void {+ if (mic.releaseTimer) clearTimeout(mic.releaseTimer);+ mic.releaseTimer = setTimeout(() => release(mic), KEEP_WARM_MS);+ }+}++export type MicrophoneDevice = { deviceId: string; label: string };++/** Audio inputs; labels are empty until the user has allowed the microphone once. */+export async function listMicrophones(): Promise<MicrophoneDevice[]> {+ if (!navigator.mediaDevices?.enumerateDevices) return [];+ const devices = await navigator.mediaDevices.enumerateDevices().catch(() => []);+ return devices+ .filter((device) => device.kind === "audioinput" && device.deviceId !== "default" && device.deviceId !== "communications")+ .map((device, index) => ({ deviceId: device.deviceId, label: device.label || `Microphone ${index + 1}` }));+}++export type MicPermission = "granted" | "denied" | "prompt" | "unknown";++/**+ * macOS asks once per app; before the first recording the desktop shell asks+ * the OS (a denied app must be switched on in System Settings). Elsewhere+ * getUserMedia itself decides.+ */+export async function ensureMicrophonePermission(): Promise<MicPermission> {+ const system = window.__OMNIRUSH_ELECTRON__?.system;+ if (!system?.getMicrophoneStatus) return "unknown";+ const { status } = await system.getMicrophoneStatus();+ if (status === "granted" || status === "not-mac") return "granted";+ if (status === "denied" || status === "restricted") return "denied";+ if (!system.askMicrophoneAccess) return "prompt";+ const answer = await system.askMicrophoneAccess();+ return answer.granted ? "granted" : "denied";+}++export function microphoneSettingsUrl(): string | null {+ const platform = navigator.userAgent;+ if (/Macintosh|Mac OS X/.test(platform)) return "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone";+ if (/Windows/.test(platform)) return "ms-settings:privacy-microphone";+ return null;+}++export async function openMicrophoneSettings(): Promise<void> {+ const url = microphoneSettingsUrl();+ const openExternal = window.__OMNIRUSH_ELECTRON__?.shell?.openExternal;+ if (url && openExternal) await openExternal(url);+}diff --git a/apps/app/src/react-app/domains/voice/use-voice-dictation.ts b/apps/app/src/react-app/domains/voice/use-voice-dictation.tsnew file mode 100644index 00000000..ca9fbb88--- /dev/null+++ b/apps/app/src/react-app/domains/voice/use-voice-dictation.ts@@ -0,0 +1,289 @@+import { useCallback, useEffect, useMemo, useRef, useState, type RefObject } from "react";++import { toast } from "@/components/ui/sonner";+import type { OmniRushServerClient, OmniRushVoiceStatus } from "@/app/lib/omnirush-server";+import {+ CircuitBreaker,+ RemoteTranscriber,+ VoiceSession,+ buildKeyterms,+ isVoiceError,+ type VoiceError,+ type VoiceSnapshot,+} from "@/app/lib/voice-core";+import { defaultHotkey, hotkeyDown, hotkeyUp, matchesHotkeyDown, matchesHotkeyUp, parseHotkey } from "./hotkey";+import { MicrophoneSource, ensureMicrophonePermission, microphoneSettingsUrl, type MicPermission } from "./mic-source";+import { useVoiceSettings, voicePrivacyNote } from "./voice-settings";+import { peekVoiceStatus, refreshVoiceStatus, useVoiceStatus } from "./voice-status";++export type VoiceClient = Pick<OmniRushServerClient, "baseUrl" | "getVoiceStatus" | "voiceTranscriptionEndpoint">;++/** What the composer's editor offers dictation (LexicalPromptEditorHandle's voice methods). */+export type VoiceEditor = {+ voiceBegin: () => void;+ voiceUpdate: (text: string, pending: boolean) => boolean;+ voiceCommit: (text: string) => string;+ voiceCancel: () => void;+};++export type VoiceAvailability =+ | { state: "hidden" }+ | { state: "ready" }+ | { state: "disabled"; reason: string };++/** Shared by every composer: three failures in 10 s pause voice for 30 s. */+const breaker = new CircuitBreaker();+/** Only one composer listens to the hotkey: the one focused last. */+let hotkeyOwner: symbol | null = null;+/** Tap mode (or a tap in "both") stops by itself after this much silence. */+const TAP_SILENCE_STOP_MS = 15_000;+const AUTO_SEND_MIN_WORDS = 3;++function availabilityFrom(enabled: boolean, client: VoiceClient | null, status: OmniRushVoiceStatus | null): VoiceAvailability {+ if (!enabled || !client) return { state: "hidden" };+ if (status && !status.signedIn) return { state: "disabled", reason: "Sign in to omnirush.ai to use voice input." };+ if (status?.available === false) {+ return { state: "disabled", reason: "Voice input isn't available on your account yet." };+ }+ return { state: "ready" };+}++function errorToast(error: VoiceError, partial: boolean): void {+ if (error.kind === "no_speech" || error.kind === "no_signal") {+ toast.info(error.message);+ return;+ }+ toast.error(partial ? `Part of the dictation was lost: ${error.message}` : error.message);+}++export type VoiceDictation = {+ availability: VoiceAvailability;+ snapshot: VoiceSnapshot | null;+ active: boolean;+ permissionDenied: boolean;+ settingsUrl: string | null;+ dismissPermission: () => void;+ toggle: () => void;+ start: (trigger: "button" | "tap" | "hold") => Promise<void>;+ stop: () => Promise<void>;+ cancel: () => void;+ /** Call when the composer gains focus: it becomes the hotkey's target. */+ claimHotkey: () => void;+};++/**+ * Dictation for one composer: availability, capture, the hotkey (hold or+ * tap), Esc to cancel, and the live transcript in the editor.+ */+export function useVoiceDictation(input: {+ client: VoiceClient | null;+ workspaceId: string | null | undefined;+ recentFiles: readonly string[];+ editorRef: RefObject<VoiceEditor | null>;+ rootRef: RefObject<HTMLElement | null>;+ disabled: boolean;+ /** Dictation ended with auto-send on and enough words: `prompt` is the draft to send. */+ onAutoSend: (prompt: string) => void;+}): VoiceDictation {+ const settings = useVoiceSettings();+ const status = useVoiceStatus(input.client, input.workspaceId, settings.enabled);+ const availability = availabilityFrom(settings.enabled, input.client, status);+ const [snapshot, setSnapshot] = useState<VoiceSnapshot | null>(null);+ const [permissionDenied, setPermissionDenied] = useState(false);+ const sessionRef = useRef<VoiceSession | null>(null);+ const pressRef = useRef<{ at: number; hold: boolean } | null>(null);+ const identity = useMemo(() => Symbol("voice-composer"), []);+ const latest = useRef(input);+ latest.current = input;+ const active = snapshot !== null && ["arming", "recording", "finalizing"].includes(snapshot.phase);++ const finish = useCallback(() => {+ sessionRef.current = null;+ pressRef.current = null;+ setSnapshot(null);+ }, []);++ const start = useCallback(async (trigger: "button" | "tap" | "hold") => {+ const { client, editorRef } = latest.current;+ if (sessionRef.current || latest.current.disabled || !client) return;+ if (availability.state !== "ready") {+ if (availability.state === "disabled") toast.info(availability.reason);+ return;+ }+ const editor = editorRef.current;+ if (!editor) return;+ const current = useVoiceSettings.getState();+ if (!current.disclosureSeen) {+ toast.info(voicePrivacyNote(peekVoiceStatus(client, latest.current.workspaceId)?.providerRetentionDays), { duration: 10_000 });+ current.update({ disclosureSeen: true });+ }+ const permission = await ensureMicrophonePermission().catch((): MicPermission => "unknown");+ if (permission === "denied") {+ setPermissionDenied(true);+ return;+ }+ const endpoint = client.voiceTranscriptionEndpoint();+ const projectStatus = peekVoiceStatus(client, latest.current.workspaceId);+ const session = new VoiceSession({+ source: new MicrophoneSource(current.deviceId),+ transcriber: new RemoteTranscriber({ url: endpoint.url, headers: () => endpoint.headers }),+ language: current.language === "auto" ? null : current.language,+ keyterms: buildKeyterms({ repo: projectStatus?.repo, branch: projectStatus?.branch, files: latest.current.recentFiles.slice(0, 20) }),+ breaker,+ silenceAutoStopMs: current.mode === "hold" ? null : TAP_SILENCE_STOP_MS,+ onAutoStop: () => void stopRef.current(),+ onChange: (next) => {+ if (sessionRef.current !== session) return;+ if (next.phase === "error") {+ // The microphone went away mid-recording.+ latest.current.editorRef.current?.voiceCancel();+ finish();+ if (next.error) toast.error(next.error.message);+ return;+ }+ setSnapshot(next);+ if (next.phase === "recording" || next.phase === "finalizing") {+ const present = latest.current.editorRef.current?.voiceUpdate(next.text, next.phase === "recording" || next.pending > 0);+ if (present === false) latest.current.editorRef.current?.voiceBegin();+ }+ },+ });+ sessionRef.current = session;+ hotkeyOwner = identity;+ editor.voiceBegin();+ try {+ await session.start();+ } catch (error) {+ editor.voiceCancel();+ finish();+ if (isVoiceError(error) && error.kind === "permission_denied") setPermissionDenied(true);+ else if (error instanceof Error && error.message) toast.error(error.message);+ }+ }, [availability, finish, identity]);++ const stop = useCallback(async () => {+ const session = sessionRef.current;+ if (!session) return;+ const result = await session.stop();+ if (sessionRef.current !== session) return;+ const editor = latest.current.editorRef.current;+ const prompt = editor ? editor.voiceCommit(result.text) : "";+ finish();+ if (result.error) {+ errorToast(result.error, Boolean(result.text));+ if (result.error.kind === "unavailable" || result.error.kind === "signed_out") {+ refreshVoiceStatus(latest.current.client, latest.current.workspaceId);+ }+ }+ if (useVoiceSettings.getState().autoSend && result.words >= AUTO_SEND_MIN_WORDS && !result.error) {+ latest.current.onAutoSend(prompt);+ }+ }, [finish]);++ const stopRef = useRef(stop);+ stopRef.current = stop;++ const cancel = useCallback(() => {+ const session = sessionRef.current;+ if (!session) return;+ session.cancel();+ latest.current.editorRef.current?.voiceCancel();+ finish();+ }, [finish]);++ const toggle = useCallback(() => {+ if (sessionRef.current) void stop();+ else void start("button");+ }, [start, stop]);++ // Esc cancels and restores the prompt; it does nothing else (no agent stop, no menu).+ useEffect(() => {+ if (!active) return;+ const onKey = (event: KeyboardEvent) => {+ if (event.key !== "Escape") return;+ event.preventDefault();+ event.stopPropagation();+ cancel();+ };+ window.addEventListener("keydown", onKey, true);+ return () => window.removeEventListener("keydown", onKey, true);+ }, [active, cancel]);++ // The hotkey: a long press is push-to-talk, a quick press toggles (per mode).+ useEffect(() => {+ if (availability.state === "hidden") return;+ const hotkey = parseHotkey(settings.hotkey ?? defaultHotkey()) ?? parseHotkey(defaultHotkey());+ if (!hotkey) return;+ hotkeyOwner ??= identity;+ const owns = () => {+ if (hotkeyOwner === identity) return true;+ // The owner went away (unmounted): the first composer still on screen takes over.+ const root = latest.current.rootRef.current;+ if (!root?.isConnected) return false;+ hotkeyOwner = identity;+ return true;+ };+ const onDown = (event: KeyboardEvent) => {+ if (!matchesHotkeyDown(event, hotkey) || !owns()) return;+ event.preventDefault();+ event.stopPropagation();+ if (event.repeat) return;+ const mode = useVoiceSettings.getState().mode;+ const action = hotkeyDown(mode, Boolean(sessionRef.current));+ if (action === "stop") {+ void stop();+ } else if (action === "start") {+ pressRef.current = { at: performance.now(), hold: true };+ void start(mode === "tap" ? "tap" : "hold");+ }+ };+ const onUp = (event: KeyboardEvent) => {+ if (!matchesHotkeyUp(event, hotkey) || !pressRef.current || !owns()) return;+ event.preventDefault();+ const heldMs = performance.now() - pressRef.current.at;+ const mode = useVoiceSettings.getState().mode;+ const action = hotkeyUp(mode, Boolean(sessionRef.current), heldMs);+ pressRef.current = null;+ if (action === "stop") void stop();+ };+ const onBlur = () => {+ // Letting go outside the window: a hold ends like a release.+ if (pressRef.current && sessionRef.current && useVoiceSettings.getState().mode !== "tap") {+ const heldMs = performance.now() - pressRef.current.at;+ pressRef.current = null;+ if (hotkeyUp(useVoiceSettings.getState().mode, true, heldMs) === "stop") void stop();+ }+ };+ window.addEventListener("keydown", onDown, true);+ window.addEventListener("keyup", onUp, true);+ window.addEventListener("blur", onBlur);+ return () => {+ window.removeEventListener("keydown", onDown, true);+ window.removeEventListener("keyup", onUp, true);+ window.removeEventListener("blur", onBlur);+ if (hotkeyOwner === identity) hotkeyOwner = null;+ };+ }, [availability.state, identity, settings.hotkey, start, stop]);++ // Leaving the composer (session switch, unmount) drops a recording in progress.+ useEffect(() => () => {+ sessionRef.current?.cancel();+ sessionRef.current = null;+ }, []);++ return {+ availability,+ snapshot,+ active,+ permissionDenied,+ settingsUrl: microphoneSettingsUrl(),+ dismissPermission: () => setPermissionDenied(false),+ toggle,+ start,+ stop,+ cancel,+ claimHotkey: () => {+ hotkeyOwner = identity;+ },+ };+}diff --git a/apps/app/src/react-app/domains/voice/voice-button.tsx b/apps/app/src/react-app/domains/voice/voice-button.tsxnew file mode 100644index 00000000..26a5e8e5--- /dev/null+++ b/apps/app/src/react-app/domains/voice/voice-button.tsx@@ -0,0 +1,124 @@+/** @jsxImportSource react */+import { LoaderCircle, Mic, MicOff, Square, X } from "lucide-react";++import { defaultHotkey, formatHotkey } from "./hotkey";+import { openMicrophoneSettings } from "./mic-source";+import type { VoiceDictation } from "./use-voice-dictation";+import { useVoiceSettings } from "./voice-settings";++function clock(ms: number): string {+ const seconds = Math.floor(ms / 1000);+ return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;+}++/** The mic beside the paperclip; while dictating, a pill with the live waveform, a timer, stop and cancel. */+export function VoiceButton(props: { voice: VoiceDictation; disabled?: boolean }) {+ const { voice } = props;+ const hotkey = useVoiceSettings((state) => state.hotkey) ?? defaultHotkey();+ if (voice.availability.state === "hidden") return null;++ if (voice.active && voice.snapshot) {+ const { snapshot } = voice;+ const finalizing = snapshot.phase === "finalizing";+ return (+ <div+ data-testid="voice-recording"+ data-voice-phase={snapshot.phase}+ className="inline-flex h-9 max-h-9 shrink-0 items-center gap-1.5 rounded-full border border-red-6/50 bg-red-3/40 pl-2.5 pr-1 text-[12px] font-medium text-red-11"+ role="status"+ aria-live="polite"+ >+ {finalizing ? (+ <LoaderCircle size={13} className="animate-spin" aria-hidden />+ ) : (+ <span className="h-2 w-2 shrink-0 animate-pulse rounded-full bg-red-9" aria-hidden />+ )}+ <span className="flex h-5 items-center gap-[2px]" data-testid="voice-waveform" aria-hidden>+ {snapshot.levels.map((level, index) => (+ <span+ // Fixed 16 bars, so the index is a stable key.+ key={index}+ className="w-[2px] rounded-full bg-red-9 transition-[height] duration-75"+ style={{ height: `${Math.max(2, Math.round(level * 20))}px` }}+ />+ ))}+ </span>+ <span className="min-w-[2.2rem] tabular-nums" data-testid="voice-timer">+ {finalizing ? "…" : clock(snapshot.elapsedMs)}+ </span>+ <span className="sr-only">{finalizing ? "Transcribing" : "Recording"}</span>+ <button+ type="button"+ className="inline-flex h-7 w-7 items-center justify-center rounded-full hover:bg-red-4"+ onClick={() => void voice.stop()}+ disabled={finalizing}+ aria-label="Stop dictation"+ title="Stop dictation"+ data-testid="voice-stop"+ >+ <Square size={11} fill="currentColor" />+ </button>+ <button+ type="button"+ className="inline-flex h-7 w-7 items-center justify-center rounded-full hover:bg-red-4"+ onClick={voice.cancel}+ aria-label="Cancel dictation (Esc)"+ title="Cancel dictation (Esc)"+ data-testid="voice-cancel"+ >+ <X size={13} />+ </button>+ </div>+ );+ }++ const unavailable = voice.availability.state === "disabled";+ const title = unavailable && voice.availability.state === "disabled"+ ? voice.availability.reason+ : `Dictate: hold or tap ${formatHotkey(hotkey)}`;+ return (+ <button+ type="button"+ data-testid="voice-mic-button"+ data-voice-available={unavailable ? "false" : "true"}+ className={`inline-flex h-9 max-h-9 w-9 shrink-0 items-center justify-center rounded-md text-gray-10 transition-colors hover:bg-gray-3 ${+ unavailable || props.disabled ? "cursor-not-allowed opacity-60" : ""+ }`}+ onClick={() => {+ if (unavailable || props.disabled) return;+ voice.toggle();+ }}+ aria-disabled={unavailable || props.disabled}+ aria-label={title}+ title={title}+ >+ {unavailable ? <MicOff size={16} /> : <Mic size={16} />}+ </button>+ );+}++/** Shown when the OS or the user blocked the microphone. */+export function VoicePermissionCard(props: { voice: VoiceDictation }) {+ if (!props.voice.permissionDenied) return null;+ const url = props.voice.settingsUrl;+ return (+ <div+ className="mx-4 mt-3 flex items-center gap-3 rounded-xl border border-amber-7/40 bg-amber-2/40 px-3 py-2 text-xs text-amber-11"+ data-testid="voice-permission-card"+ role="alert"+ >+ <span className="min-w-0 flex-1">+ Microphone access is off for OmniRush.ai.{" "}+ {url ? "Turn it on in your system's privacy settings, then try again." : "Check that a microphone is connected and allowed, then try again."}+ </span>+ {url ? (+ <button type="button" className="shrink-0 font-medium hover:underline" onClick={() => void openMicrophoneSettings()}>+ Open settings+ </button>+ ) : null}+ <button type="button" className="shrink-0 font-medium hover:underline" onClick={props.voice.dismissPermission}>+ Dismiss+ </button>+ </div>+ );+}diff --git a/apps/app/src/react-app/domains/voice/voice-settings-view.tsx b/apps/app/src/react-app/domains/voice/voice-settings-view.tsxnew file mode 100644index 00000000..7225fbd6--- /dev/null+++ b/apps/app/src/react-app/domains/voice/voice-settings-view.tsx@@ -0,0 +1,278 @@+/** @jsxImportSource react */+import { useEffect, useRef, useState } from "react";++import { Button } from "@/components/ui/button";+import { Select, SelectContent, SelectGroup, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";+import { Separator } from "@/components/ui/separator";+import { Switch } from "@/components/ui/switch";+import { levelFromRms, rms } from "@/app/lib/voice-core";+import {+ LayoutSection,+ LayoutSectionDescription,+ LayoutSectionHeader,+ LayoutSectionItem,+ LayoutSectionItemDescription,+ LayoutSectionItemHeader,+ LayoutSectionItemHeaderActions,+ LayoutSectionItemTitle,+ LayoutSectionTitle,+ LayoutStack,+} from "@/react-app/domains/settings/settings-layout";+import { defaultHotkey, formatHotkey, hotkeyFromEvent } from "./hotkey";+import { MicrophoneSource, listMicrophones, microphoneSettingsUrl, openMicrophoneSettings, releaseMicrophone, type MicrophoneDevice } from "./mic-source";+import type { VoiceClient } from "./use-voice-dictation";+import { useVoiceStatus } from "./voice-status";+import { VOICE_LANGUAGES, useVoiceSettings, voicePrivacyNote, type VoiceMode } from "./voice-settings";++const MODES: ReadonlyArray<{ value: VoiceMode; label: string }> = [+ { value: "both", label: "Hold or tap" },+ { value: "hold", label: "Hold to talk" },+ { value: "tap", label: "Tap to start and stop" },+];+const SYSTEM_DEFAULT = "system-default";++function MicTest(props: { deviceId: string | null }) {+ const [level, setLevel] = useState<number | null>(null);+ const [error, setError] = useState<string | null>(null);+ const sourceRef = useRef<MicrophoneSource | null>(null);+ const stop = () => {+ sourceRef.current?.stop();+ sourceRef.current = null;+ setLevel(null);+ };+ useEffect(() => stop, []);+ const start = async () => {+ setError(null);+ const source = new MicrophoneSource(props.deviceId);+ sourceRef.current = source;+ try {+ // Levels only: the test audio is measured in memory and dropped.+ await source.start((frames) => setLevel(levelFromRms(rms(frames))));+ setTimeout(() => {+ if (sourceRef.current === source) stop();+ }, 6_000);+ } catch (caught) {+ sourceRef.current = null;+ setError(caught instanceof Error ? caught.message : String(caught));+ }+ };+ return (+ <div className="flex items-center gap-3">+ <div className="h-2 w-32 overflow-hidden rounded-full bg-gray-4" aria-label="Microphone level" data-testid="voice-settings-meter">+ <div className="h-full rounded-full bg-green-9 transition-[width] duration-75" style={{ width: `${Math.round((level ?? 0) * 100)}%` }} />+ </div>+ <Button variant="outline" size="sm" onClick={() => (level === null ? void start() : stop())}>+ {level === null ? "Test" : "Stop"}+ </Button>+ {error ? <span className="text-xs text-red-11">{error}</span> : null}+ </div>+ );+}++export function VoiceSettingsView(props: { client: VoiceClient | null }) {+ const settings = useVoiceSettings();+ const [devices, setDevices] = useState<MicrophoneDevice[]>([]);+ const [recording, setRecording] = useState(false);+ const status = useVoiceStatus(props.client, null);++ useEffect(() => {+ void listMicrophones().then(setDevices);+ const refresh = () => void listMicrophones().then(setDevices);+ navigator.mediaDevices?.addEventListener?.("devicechange", refresh);+ return () => navigator.mediaDevices?.removeEventListener?.("devicechange", refresh);+ }, []);++ useEffect(() => {+ if (!recording) return;+ const onKey = (event: KeyboardEvent) => {+ event.preventDefault();+ event.stopPropagation();+ if (event.key === "Escape") {+ setRecording(false);+ return;+ }+ const value = hotkeyFromEvent(event);+ if (!value) return;+ settings.update({ hotkey: value });+ setRecording(false);+ };+ window.addEventListener("keydown", onKey, true);+ return () => window.removeEventListener("keydown", onKey, true);+ }, [recording, settings]);++ const hotkey = settings.hotkey ?? defaultHotkey();+ const availability = !props.client+ ? null+ : status && !status.signedIn+ ? "Sign in to omnirush.ai to use voice input."+ : status?.available === false+ ? "Voice input isn't available on your account yet."+ : null;+ const deviceKnown = settings.deviceId === null || devices.some((device) => device.deviceId === settings.deviceId);++ return (+ <LayoutStack>+ <LayoutSection>+ <LayoutSectionHeader>+ <LayoutSectionTitle>Voice input</LayoutSectionTitle>+ <LayoutSectionDescription>+ Dictate prompts instead of typing. Your words appear in the composer, where you can edit them before sending.+ </LayoutSectionDescription>+ </LayoutSectionHeader>++ <LayoutSectionItem>+ <LayoutSectionItemHeader>+ <LayoutSectionItemTitle>Dictation</LayoutSectionItemTitle>+ <LayoutSectionItemDescription>+ {availability ?? "Show the microphone button in the composer and listen for the hotkey."}+ </LayoutSectionItemDescription>+ <LayoutSectionItemHeaderActions>+ <Switch+ aria-label="Dictation"+ data-testid="voice-settings-enabled"+ checked={settings.enabled}+ onCheckedChange={(enabled) => {+ settings.update({ enabled });+ if (!enabled) releaseMicrophone();+ }}+ />+ </LayoutSectionItemHeaderActions>+ </LayoutSectionItemHeader>+ </LayoutSectionItem>++ <LayoutSectionItem>+ <LayoutSectionItemHeader>+ <LayoutSectionItemTitle>Hotkey</LayoutSectionItemTitle>+ <LayoutSectionItemDescription>Works whenever the OmniRush.ai window is focused.</LayoutSectionItemDescription>+ <LayoutSectionItemHeaderActions>+ <div className="flex items-center gap-2">+ <Button variant="outline" size="sm" data-testid="voice-settings-hotkey" onClick={() => setRecording(true)}>+ {recording ? "Press a shortcut…" : formatHotkey(hotkey)}+ </Button>+ {settings.hotkey ? (+ <Button variant="ghost" size="sm" onClick={() => settings.update({ hotkey: null })}>+ Reset+ </Button>+ ) : null}+ </div>+ </LayoutSectionItemHeaderActions>+ </LayoutSectionItemHeader>+ </LayoutSectionItem>++ <LayoutSectionItem>+ <LayoutSectionItemHeader>+ <LayoutSectionItemTitle>Hold or tap</LayoutSectionItemTitle>+ <LayoutSectionItemDescription>+ Hold the hotkey while you talk, or tap it to start and tap again to stop. Esc cancels and restores your prompt.+ </LayoutSectionItemDescription>+ <LayoutSectionItemHeaderActions>+ <div className="w-56 max-w-full">+ <Select value={settings.mode} items={MODES} onValueChange={(value) => value && settings.update({ mode: value })}>+ <SelectTrigger className="w-full" aria-label="Hold or tap">+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ <SelectGroup>+ {MODES.map((mode) => (+ <SelectItem key={mode.value} value={mode.value}>{mode.label}</SelectItem>+ ))}+ </SelectGroup>+ </SelectContent>+ </Select>+ </div>+ </LayoutSectionItemHeaderActions>+ </LayoutSectionItemHeader>+ </LayoutSectionItem>++ <LayoutSectionItem>+ <LayoutSectionItemHeader>+ <LayoutSectionItemTitle>Input device</LayoutSectionItemTitle>+ <LayoutSectionItemDescription>+ {deviceKnown ? "The microphone used for dictation." : "The chosen microphone is not connected; the system default is used."}+ </LayoutSectionItemDescription>+ <LayoutSectionItemHeaderActions>+ <div className="flex flex-wrap items-center gap-3">+ <div className="w-56 max-w-full">+ <Select+ value={settings.deviceId ?? SYSTEM_DEFAULT}+ items={[{ value: SYSTEM_DEFAULT, label: "System default" }, ...devices.map((device) => ({ value: device.deviceId, label: device.label }))]}+ onValueChange={(value) => {+ settings.update({ deviceId: !value || value === SYSTEM_DEFAULT ? null : value });+ releaseMicrophone();+ }}+ >+ <SelectTrigger className="w-full" aria-label="Input device">+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ <SelectGroup>+ <SelectItem value={SYSTEM_DEFAULT}>System default</SelectItem>+ {devices.map((device) => (+ <SelectItem key={device.deviceId} value={device.deviceId}>{device.label}</SelectItem>+ ))}+ </SelectGroup>+ </SelectContent>+ </Select>+ </div>+ <MicTest deviceId={deviceKnown ? settings.deviceId : null} />+ </div>+ </LayoutSectionItemHeaderActions>+ </LayoutSectionItemHeader>+ </LayoutSectionItem>++ <LayoutSectionItem>+ <LayoutSectionItemHeader>+ <LayoutSectionItemTitle>Language</LayoutSectionItemTitle>+ <LayoutSectionItemDescription>The language you speak. Automatic detection works for most people.</LayoutSectionItemDescription>+ <LayoutSectionItemHeaderActions>+ <div className="w-56 max-w-full">+ <Select value={settings.language} items={VOICE_LANGUAGES} onValueChange={(value) => value && settings.update({ language: value })}>+ <SelectTrigger className="w-full" aria-label="Dictation language">+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ <SelectGroup>+ {VOICE_LANGUAGES.map((language) => (+ <SelectItem key={language.value} value={language.value}>{language.label}</SelectItem>+ ))}+ </SelectGroup>+ </SelectContent>+ </Select>+ </div>+ </LayoutSectionItemHeaderActions>+ </LayoutSectionItemHeader>+ </LayoutSectionItem>++ <LayoutSectionItem>+ <LayoutSectionItemHeader>+ <LayoutSectionItemTitle>Send after dictation</LayoutSectionItemTitle>+ <LayoutSectionItemDescription>Send the prompt as soon as you stop talking (at least three words). Off: review and edit first.</LayoutSectionItemDescription>+ <LayoutSectionItemHeaderActions>+ <Switch aria-label="Send after dictation" checked={settings.autoSend} onCheckedChange={(autoSend) => settings.update({ autoSend })} />+ </LayoutSectionItemHeaderActions>+ </LayoutSectionItemHeader>+ </LayoutSectionItem>+ </LayoutSection>++ <Separator />++ <LayoutSection>+ <LayoutSectionHeader>+ <LayoutSectionTitle>Privacy</LayoutSectionTitle>+ <LayoutSectionDescription data-testid="voice-privacy-note">{voicePrivacyNote(status?.providerRetentionDays)}</LayoutSectionDescription>+ </LayoutSectionHeader>+ {microphoneSettingsUrl() ? <LayoutSectionItem>+ <LayoutSectionItemHeader>+ <LayoutSectionItemTitle>Microphone permission</LayoutSectionItemTitle>+ <LayoutSectionItemDescription>If dictation says the microphone is off, allow OmniRush.ai in your system's privacy settings.</LayoutSectionItemDescription>+ <LayoutSectionItemHeaderActions>+ <Button variant="outline" size="sm" onClick={() => void openMicrophoneSettings()}>+ Open privacy settings+ </Button>+ </LayoutSectionItemHeaderActions>+ </LayoutSectionItemHeader>+ </LayoutSectionItem> : null}+ </LayoutSection>+ </LayoutStack>+ );+}diff --git a/apps/app/src/react-app/domains/voice/voice-settings.ts b/apps/app/src/react-app/domains/voice/voice-settings.tsnew file mode 100644index 00000000..c4939bc4--- /dev/null+++ b/apps/app/src/react-app/domains/voice/voice-settings.ts@@ -0,0 +1,84 @@+import { create } from "zustand";+import { createJSONStorage, persist } from "zustand/middleware";++/** hold: talk while the hotkey is held. tap: press to start, press again to stop. both: a long press holds, a quick press taps. */+export type VoiceMode = "both" | "hold" | "tap";++export type VoiceSettings = {+ enabled: boolean;+ mode: VoiceMode;+ /** The hotkey as modifiers + KeyboardEvent.code, e.g. "Ctrl+Space" / "Alt+Space"; null for the platform default. */+ hotkey: string | null;+ /** An audio input's deviceId; null follows the system default. */+ deviceId: string | null;+ /** BCP-47 language hint; "auto" lets the service detect it. */+ language: string;+ /** Send the prompt when dictation ends (at least 3 words). */+ autoSend: boolean;+ /** The one-time privacy note was shown. */+ disclosureSeen: boolean;+};++type VoiceSettingsStore = VoiceSettings & {+ update: (patch: Partial<VoiceSettings>) => void;+};++export const DEFAULT_VOICE_SETTINGS: VoiceSettings = {+ enabled: true,+ mode: "both",+ hotkey: null,+ deviceId: null,+ language: "auto",+ autoSend: false,+ disclosureSeen: false,+};++export const useVoiceSettings = create<VoiceSettingsStore>()(+ persist(+ (set) => ({+ ...DEFAULT_VOICE_SETTINGS,+ update: (patch) => set(patch),+ }),+ {+ name: "omnirush.voice.v1",+ storage: createJSONStorage(() => localStorage),+ partialize: ({ update: _update, ...settings }) => settings,+ },+ ),+);++/**+ * Shown once, on first use, and in Settings → Voice. The provider's+ * retention comes from the backend's voice status (it cannot be turned off+ * upstream today: 30 days); without one the clause is left out.+ */+export function voicePrivacyNote(providerRetentionDays?: number | null): string {+ const provider = providerRetentionDays+ ? `omnirush.ai doesn't store your audio; the provider keeps a copy for up to ${providerRetentionDays} days.`+ : "omnirush.ai doesn't store your audio.";+ return `Voice is transcribed by our cloud speech provider. ${provider} Nothing is saved on this computer, and only the text you send becomes part of your prompt.`;+}++/** Languages offered as a hint; the service detects the language on its own with "auto". */+export const VOICE_LANGUAGES: ReadonlyArray<{ value: string; label: string }> = [+ { value: "auto", label: "Detect automatically" },+ { value: "en", label: "English" },+ { value: "es", label: "Spanish" },+ { value: "fr", label: "French" },+ { value: "de", label: "German" },+ { value: "it", label: "Italian" },+ { value: "pt", label: "Portuguese" },+ { value: "nl", label: "Dutch" },+ { value: "pl", label: "Polish" },+ { value: "ru", label: "Russian" },+ { value: "uk", label: "Ukrainian" },+ { value: "tr", label: "Turkish" },+ { value: "hi", label: "Hindi" },+ { value: "ja", label: "Japanese" },+ { value: "ko", label: "Korean" },+ { value: "zh", label: "Chinese" },+ { value: "vi", label: "Vietnamese" },+ { value: "th", label: "Thai" },+ { value: "id", label: "Indonesian" },+ { value: "ca", label: "Catalan" },+];diff --git a/apps/app/src/react-app/domains/voice/voice-status.ts b/apps/app/src/react-app/domains/voice/voice-status.tsnew file mode 100644index 00000000..557fa299--- /dev/null+++ b/apps/app/src/react-app/domains/voice/voice-status.ts@@ -0,0 +1,69 @@+import { useEffect, useState } from "react";++import type { OmniRushServerClient, OmniRushVoiceStatus } from "@/app/lib/omnirush-server";++type StatusClient = Pick<OmniRushServerClient, "baseUrl" | "getVoiceStatus">;+type Entry = { data: OmniRushVoiceStatus | null; fetchedAt: number; inflight: Promise<void> | null };++const FRESH_MS = 60_000;+const REFRESH_MS = 5 * 60_000;+const entries = new Map<string, Entry>();+const listeners = new Set<() => void>();++function keyOf(client: StatusClient, workspaceId: string | null | undefined): string {+ return `${client.baseUrl}\u0000${workspaceId ?? ""}`;+}++function notify(): void {+ for (const listener of listeners) listener();+}++function load(client: StatusClient, workspaceId: string | null | undefined, force = false): Promise<void> {+ const key = keyOf(client, workspaceId);+ const entry = entries.get(key) ?? { data: null, fetchedAt: 0, inflight: null };+ entries.set(key, entry);+ if (entry.inflight) return entry.inflight;+ if (!force && entry.data && Date.now() - entry.fetchedAt < FRESH_MS) return Promise.resolve();+ entry.inflight = client.getVoiceStatus(workspaceId ?? null)+ .then((data) => {+ entry.data = data;+ }, () => {+ // Unknown: keep the last answer; the mic stays usable and the first upload decides.+ })+ .finally(() => {+ entry.fetchedAt = Date.now();+ entry.inflight = null;+ notify();+ });+ return entry.inflight;+}++/** The last known voice status for this server and workspace (null until the first answer). */+export function peekVoiceStatus(client: StatusClient | null, workspaceId: string | null | undefined): OmniRushVoiceStatus | null {+ return client ? entries.get(keyOf(client, workspaceId))?.data ?? null : null;+}++/** Asks the server again (after a refusal that may have changed availability). */+export function refreshVoiceStatus(client: StatusClient | null, workspaceId: string | null | undefined): void {+ if (client) void load(client, workspaceId, true);+}++/**+ * GET /omnirush/voice/status, cached per server and workspace for a minute+ * and refreshed every five, shared by every composer and Settings → Voice.+ */+export function useVoiceStatus(client: StatusClient | null, workspaceId: string | null | undefined, enabled = true): OmniRushVoiceStatus | null {+ const [, setVersion] = useState(0);+ useEffect(() => {+ if (!client || !enabled) return;+ const listener = () => setVersion((version) => version + 1);+ listeners.add(listener);+ void load(client, workspaceId);+ const timer = setInterval(() => void load(client, workspaceId), REFRESH_MS);+ return () => {+ listeners.delete(listener);+ clearInterval(timer);+ };+ }, [client, enabled, workspaceId]);+ return enabled ? peekVoiceStatus(client, workspaceId) : null;+}diff --git a/apps/app/src/react-app/shell/command-palette-settings.ts b/apps/app/src/react-app/shell/command-palette-settings.tsindex 4e94e232..0c05f116 100644--- a/apps/app/src/react-app/shell/command-palette-settings.ts+++ b/apps/app/src/react-app/shell/command-palette-settings.ts@@ -21,6 +21,7 @@ const SETTINGS_KEYWORDS: Partial<Record<SettingsTab, string[]>> = {environment: ["env", "environment variables", "secrets", "tokens", "api keys"],advanced: ["advanced settings", "runtime", "developer", "connection", "server", "port", "reset", "fix", "repair", "clean up", "troubleshoot", "recovery"],appearance: ["theme", "dark mode", "light mode", "color", "font", "look"],+ voice: ["voice", "dictation", "microphone", "mic", "speech", "talk", "hotkey"],updates: ["version", "upgrade", "check for updates", "release"],debug: ["logs", "diagnostics", "developer mode"],"cloud-account": ["sign in", "log in", "login", "account", "organization", "org", "den", "cloud", "omnirush cloud"],diff --git a/apps/app/src/react-app/shell/settings-route.tsx b/apps/app/src/react-app/shell/settings-route.tsxindex a6152e96..14bb04ab 100644--- a/apps/app/src/react-app/shell/settings-route.tsx+++ b/apps/app/src/react-app/shell/settings-route.tsx@@ -89,6 +89,7 @@ import { EffectivePermissionsPanel } from "@/react-app/domains/settings/panels/eimport { SettingsStack } from "@/react-app/domains/settings/settings-section";import { AdvancedView } from "@/react-app/domains/settings/pages/advanced-view";import { AppearanceView } from "@/react-app/domains/settings/pages/appearance-view";+import { VoiceSettingsView } from "@/react-app/domains/voice/voice-settings-view";import {connectPluginsForComposer,EMPTY_CONNECT_CAPABILITY_INVENTORY,@@ -297,6 +298,7 @@ export function parseSettingsPath(pathname: string): {case "preferences":case "permissions":case "appearance":+ case "voice":case "environment":case "updates":case "debug":@@ -2531,6 +2533,8 @@ function SettingsRouteContent(props: SettingsSurfaceProps = {}) {toggleHideTitlebar={() => setHideTitlebar((current) => !current)}/>);+ case "voice":+ return <VoiceSettingsView client={omnirushClient} />;case "updates":return (<UpdatesViewdiff --git a/apps/app/tests/command-palette-settings.test.ts b/apps/app/tests/command-palette-settings.test.tsindex cfa36b08..90ec6cbb 100644--- a/apps/app/tests/command-palette-settings.test.ts+++ b/apps/app/tests/command-palette-settings.test.ts@@ -32,6 +32,7 @@ describe("command palette settings", () => {"settings:advanced","settings:ai","settings:appearance",+ "settings:voice","settings:environment","settings:updates","settings:advanced/organization-server",diff --git a/apps/app/tests/voice-dictation-ui.test.tsx b/apps/app/tests/voice-dictation-ui.test.tsxnew file mode 100644index 00000000..df62fe5a--- /dev/null+++ b/apps/app/tests/voice-dictation-ui.test.tsx@@ -0,0 +1,329 @@+/** @jsxImportSource react */+import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test";+import { GlobalRegistrator } from "@happy-dom/global-registrator";+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";+import { createRequire } from "node:module";+import { act, useRef, useState } from "react";+import { createRoot, type Root } from "react-dom/client";++import type { OmniRushVoiceStatus } from "../src/app/lib/omnirush-server";+import type { LexicalPromptEditorHandle } from "../src/react-app/domains/session/surface/composer/editor";+import type { VoiceClient } from "../src/react-app/domains/voice/use-voice-dictation";+import { defaultHotkey, formatHotkey, hotkeyDown, hotkeyFromEvent, hotkeyUp, parseHotkey } from "../src/react-app/domains/voice/hotkey";++const RATE = 16_000;++function tone(ms: number): Int16Array {+ const out = new Int16Array(Math.round((RATE * ms) / 1000));+ for (let i = 0; i < out.length; i += 1) out[i] = Math.round(0.3 * 32767 * Math.sin((2 * Math.PI * 200 * i) / RATE));+ return out;+}+const quiet = (ms: number) => new Int16Array(Math.round((RATE * ms) / 1000));+function concat(...parts: Int16Array[]): Int16Array {+ const out = new Int16Array(parts.reduce((n, p) => n + p.length, 0));+ let at = 0;+ for (const part of parts) {+ out.set(part, at);+ at += part.length;+ }+ return out;+}++/** What the fake microphone plays next (16 kHz PCM, real time in 20 ms chunks). */+let micAudio = concat(tone(1_200), quiet(800), tone(1_200), quiet(3_000));+const micStarts: string[] = [];++describe("hotkey rules", () => {+ test("defaults: Option+Space on macOS, Ctrl+Space elsewhere", () => {+ expect(defaultHotkey(true)).toBe("Alt+Space");+ expect(defaultHotkey(false)).toBe("Ctrl+Space");+ expect(formatHotkey("Alt+Space", true)).toBe("⌥Space");+ expect(formatHotkey("Ctrl+Space", false)).toBe("Ctrl+Space");+ });++ test("a bare key is refused (it would fire while typing); modifier combos and F-keys are accepted", () => {+ expect(parseHotkey("Space")).toBeNull();+ expect(parseHotkey("KeyM")).toBeNull();+ expect(parseHotkey("F8")).not.toBeNull();+ expect(parseHotkey("Ctrl+Shift+KeyM")).toEqual({ ctrl: true, alt: false, shift: true, meta: false, code: "KeyM" });+ expect(hotkeyFromEvent({ ctrlKey: false, altKey: true, shiftKey: false, metaKey: false, code: "Space" })).toBe("Alt+Space");+ expect(hotkeyFromEvent({ ctrlKey: true, altKey: false, shiftKey: false, metaKey: false, code: "ControlLeft" })).toBeNull();+ });++ test("hold versus tap", () => {+ expect(hotkeyDown("both", false)).toBe("start");+ expect(hotkeyUp("both", true, 600)).toBe("stop"); // held: push-to-talk+ expect(hotkeyUp("both", true, 120)).toBe("none"); // tapped: keeps recording+ expect(hotkeyDown("both", true)).toBe("stop"); // second tap+ expect(hotkeyUp("hold", true, 50)).toBe("stop");+ expect(hotkeyDown("hold", true)).toBe("none");+ expect(hotkeyUp("tap", true, 900)).toBe("none");+ expect(hotkeyDown("tap", true)).toBe("stop");+ });+});++describe("dictation in the composer", () => {+ let root: Root;+ let container: HTMLDivElement;+ let queryClient: QueryClient;+ let status: OmniRushVoiceStatus;+ let uploads: FormData[];+ let answers: (index: number) => Response;+ let sent: string[];+ let Harness: (props: { client: VoiceClient; initial: string }) => React.JSX.Element;++ beforeAll(async () => {+ const require = createRequire(import.meta.url);+ for (const moduleId of [+ "lexical",+ "@lexical/react/LexicalComposer.js",+ "@lexical/react/LexicalPlainTextPlugin.js",+ "@lexical/react/LexicalContentEditable.js",+ "@lexical/react/LexicalErrorBoundary.js",+ "@lexical/react/LexicalOnChangePlugin.js",+ "@lexical/react/LexicalHistoryPlugin.js",+ "@lexical/react/LexicalComposerContext.js",+ ]) {+ const moduleExports = require(moduleId);+ mock.module(moduleId, () => moduleExports);+ }+ if (typeof globalThis.document === "undefined") GlobalRegistrator.register({ url: "http://localhost/" });+ Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", { configurable: true, value: true });+ // Recording updates arrive from audio timers, outside act(); the warning is noise here.+ const consoleError = console.error;+ console.error = (...args: unknown[]) => {+ if (typeof args[0] === "string" && args[0].includes("not wrapped in act")) return;+ consoleError(...args);+ };+ const core = await import("../src/app/lib/voice-core");+ const actualMic = await import("../src/react-app/domains/voice/mic-source");+ // The microphone: plays `micAudio` through voice-core's file source.+ mock.module("../src/react-app/domains/voice/mic-source", () => ({+ ...actualMic,+ MicrophoneSource: class {+ private inner = new core.WavFileSource(core.encodeWav(micAudio), { realtime: true, chunkMs: 20 });+ constructor(deviceId: string | null) {+ micStarts.push(String(deviceId));+ }+ start(onFrames: (frames: Int16Array) => void, onEnd?: () => void) {+ return this.inner.start(onFrames, onEnd);+ }+ stop() {+ this.inner.stop();+ }+ },+ }));+ const fetchStub = async (input: RequestInfo | URL, init?: RequestInit) => {+ const url = String(input);+ if (!url.endsWith("/omnirush/voice/transcribe")) return new Response("{}", { status: 404 });+ const form = init?.body as FormData;+ uploads.push(form);+ await new Promise((resolve) => setTimeout(resolve, 30));+ return answers(Number(form.get("segment_index")));+ };+ Object.defineProperty(globalThis, "fetch", { configurable: true, value: fetchStub });+ const { LexicalPromptEditor } = await import("../src/react-app/domains/session/surface/composer/editor");+ const { useVoiceDictation } = await import("../src/react-app/domains/voice/use-voice-dictation");+ const { VoiceButton, VoicePermissionCard } = await import("../src/react-app/domains/voice/voice-button");+ Harness = function VoiceHarness(props) {+ const [draft, setDraft] = useState(props.initial);+ const editorRef = useRef<LexicalPromptEditorHandle | null>(null);+ const rootRef = useRef<HTMLDivElement | null>(null);+ const voice = useVoiceDictation({+ client: props.client,+ workspaceId: "ws-1",+ recentFiles: ["src/auth.ts"],+ editorRef,+ rootRef,+ disabled: false,+ onAutoSend: (prompt) => sent.push(prompt),+ });+ return (+ <div ref={rootRef}>+ <output data-testid="draft">{draft}</output>+ <VoicePermissionCard voice={voice} />+ <LexicalPromptEditor+ ref={editorRef}+ value={draft}+ mentions={{}}+ submitDisabled={false}+ placeholder="Ask"+ onChange={setDraft}+ onSubmit={() => {}}+ />+ <VoiceButton voice={voice} />+ </div>+ );+ };+ });++ afterAll(() => {+ mock.restore();+ });++ let testClient: VoiceClient;+ function client(): VoiceClient {+ testClient = {+ baseUrl: "http://127.0.0.1:1",+ getVoiceStatus: async () => status,+ voiceTranscriptionEndpoint: () => ({ url: "http://127.0.0.1:1/omnirush/voice/transcribe", headers: { Authorization: "Bearer t" } }),+ };+ return testClient;+ }++ async function mount(initial = "Fix the") {+ status = { signedIn: true, available: true, reason: null, repo: "omnirush-gui", branch: "feat/voice-mode" };+ uploads = [];+ sent = [];+ answers = (index) => Response.json({ text: ["oh auth middleware", "in auth dot ts"][index] ?? "" });+ micStarts.length = 0;+ micAudio = concat(tone(1_200), quiet(800), tone(1_200), quiet(3_000));+ window.localStorage.clear();+ delete (window as { __OMNIRUSH_ELECTRON__?: unknown }).__OMNIRUSH_ELECTRON__;+ queryClient = new QueryClient();+ container = document.createElement("div");+ document.body.append(container);+ root = createRoot(container);+ const { useVoiceSettings, DEFAULT_VOICE_SETTINGS } = await import("../src/react-app/domains/voice/voice-settings");+ useVoiceSettings.setState({ ...DEFAULT_VOICE_SETTINGS, disclosureSeen: true });+ await act(async () => {+ root.render(+ <QueryClientProvider client={queryClient}>+ <Harness client={client()} initial={initial} />+ </QueryClientProvider>,+ );+ });+ const { refreshVoiceStatus } = await import("../src/react-app/domains/voice/voice-status");+ refreshVoiceStatus(testClient, "ws-1");+ await waitFor(() => container.querySelector("[data-testid=voice-mic-button]")?.getAttribute("data-voice-available") === "true", "mic button");++ }++ async function unmount() {+ await act(async () => root.unmount());+ container.remove();+ }++ const $ = (id: string) => container.querySelector<HTMLElement>(`[data-testid=${id}]`);+ const draft = () => $("draft")?.textContent ?? "";++ async function waitFor(predicate: () => boolean, label: string, ms = 6_000) {+ const deadline = Date.now() + ms;+ while (Date.now() < deadline) {+ if (predicate()) return;+ await act(async () => {+ await new Promise<void>((resolve) => setTimeout(resolve, 20));+ });+ }+ throw new Error(`Timed out waiting for ${label}; draft=${JSON.stringify(draft())}`);+ }++ async function key(type: "keydown" | "keyup", init: KeyboardEventInit) {+ await act(async () => {+ window.dispatchEvent(new KeyboardEvent(type, { bubbles: true, cancelable: true, ...init }));+ });+ }++ test("tap the mic: live waveform and dim partial text, then the text lands at the cursor, editable and unsent", async () => {+ await mount("Fix the");+ await act(async () => $("voice-mic-button")!.click());+ await waitFor(() => Boolean($("voice-recording")), "recording pill");+ expect($("voice-waveform")!.children).toHaveLength(16);+ // The first segment's text shows while the user is still talking, dim, in the editor.+ await waitFor(() => container.querySelector("[data-voice-interim]")?.textContent?.includes("OAuth middleware") ?? false, "interim text");+ expect(container.querySelector("[data-voice-interim]")!.className).toContain("italic");+ await waitFor(() => uploads.length >= 2, "the second phrase uploaded");+ await act(async () => $("voice-stop")!.click());+ await waitFor(() => !$("voice-recording"), "recording ended");+ expect(draft()).toBe("Fix the OAuth middleware in auth.ts");+ expect(container.querySelector("[data-voice-interim]")).toBeNull();+ expect(uploads).toHaveLength(2);+ expect(uploads[0]!.get("prompt_terms")).toContain("omnirush-gui");+ expect(sent).toEqual([]);+ await unmount();+ });++ test("Esc cancels: uploads stop and the prompt is restored exactly", async () => {+ await mount("Keep my draft");+ answers = () => new Promise<Response>(() => {}) as unknown as Response;+ await act(async () => $("voice-mic-button")!.click());+ await waitFor(() => uploads.length > 0, "first upload");+ await key("keydown", { key: "Escape", code: "Escape" });+ await waitFor(() => !$("voice-recording"), "cancelled");+ expect(draft()).toBe("Keep my draft");+ expect(container.querySelector("[data-voice-interim]")).toBeNull();+ await unmount();+ });++ test("hold the hotkey to talk, release to finish", async () => {+ await mount("");+ const hotkey = parseHotkey(defaultHotkey())!;+ const down = { code: "Space", key: " ", ctrlKey: hotkey.ctrl, altKey: hotkey.alt };+ await key("keydown", down);+ await waitFor(() => Boolean($("voice-recording")), "recording on keydown");+ await waitFor(() => uploads.length >= 1, "a segment uploaded while held");+ await new Promise((resolve) => setTimeout(resolve, 1_300));+ await key("keyup", { code: "Space", key: " " });+ await waitFor(() => !$("voice-recording"), "released");+ expect(draft()).toBe("OAuth middleware in auth.ts");+ await unmount();+ });++ test("auto-send (when turned on) sends after three or more words", async () => {+ await mount("");+ const { useVoiceSettings } = await import("../src/react-app/domains/voice/voice-settings");+ useVoiceSettings.getState().update({ autoSend: true });+ await act(async () => $("voice-mic-button")!.click());+ await waitFor(() => uploads.length >= 2, "both segments uploaded");+ await act(async () => $("voice-stop")!.click());+ await waitFor(() => sent.length === 1, "auto-send");+ expect(sent[0]).toBe("OAuth middleware in auth.ts");+ await unmount();+ });++ test("voice unavailable on the account: the mic is disabled with a reason", async () => {+ await mount();+ status = { ...status, available: false, reason: "voice_unavailable" };+ const { refreshVoiceStatus } = await import("../src/react-app/domains/voice/voice-status");+ refreshVoiceStatus(testClient, "ws-1");+ await waitFor(() => $("voice-mic-button")?.getAttribute("data-voice-available") === "false", "disabled mic");+ expect($("voice-mic-button")!.title).toBe("Voice input isn't available on your account yet.");+ await act(async () => $("voice-mic-button")!.click());+ expect($("voice-recording")).toBeNull();+ expect(micStarts).toEqual([]);+ await unmount();+ });++ test("a 503 voice_unavailable mid-dictation ends it without inserting text", async () => {+ await mount("Start");+ answers = () => Response.json({ detail: "voice_unavailable" }, { status: 503 });+ await act(async () => $("voice-mic-button")!.click());+ await waitFor(() => uploads.length >= 1, "upload");+ await act(async () => $("voice-stop")!.click());+ await waitFor(() => !$("voice-recording"), "stopped");+ expect(draft()).toBe("Start");+ await unmount();+ });++ test("microphone permission denied by the OS shows the settings card and never opens the mic", async () => {+ await mount();+ Object.defineProperty(window, "__OMNIRUSH_ELECTRON__", {+ configurable: true,+ value: { system: { getMicrophoneStatus: async () => ({ platform: "darwin", status: "denied" }) } },+ });+ await act(async () => $("voice-mic-button")!.click());+ await waitFor(() => Boolean($("voice-permission-card")), "permission card");+ expect(micStarts).toEqual([]);+ await unmount();+ });++ test("a silent microphone uploads nothing and inserts nothing", async () => {+ await mount("Draft");+ micAudio = quiet(1_500);+ await act(async () => $("voice-mic-button")!.click());+ await waitFor(() => !$("voice-recording"), "the source ran dry and dictation ended", 8_000);+ expect(uploads).toHaveLength(0);+ expect(draft()).toBe("Draft");+ await unmount();+ });+});diff --git a/apps/desktop/electron/media-permissions.mjs b/apps/desktop/electron/media-permissions.mjsindex f52f5837..270023b5 100644--- a/apps/desktop/electron/media-permissions.mjs+++ b/apps/desktop/electron/media-permissions.mjs@@ -1,6 +1,16 @@-export function configureFakeMediaForTests(app, enabled) {- if (!enabled) return;+/**+ * Test switches for capture. OMNIRUSH_ELECTRON_FAKE_MEDIA gives Chromium a+ * fake microphone (a beep); OMNIRUSH_VOICE_INPUT_FILE (the voice-core test+ * hook) plays that WAV file through the fake microphone once, so voice+ * dictation runs end to end, AudioWorklet included, without a real device.+ */+export function configureFakeMediaForTests(app, enabled, env = process.env) {+ const audioFile = String(env.OMNIRUSH_VOICE_INPUT_FILE ?? "").trim();+ if (!enabled && !audioFile) return;app.commandLine.appendSwitch("use-fake-device-for-media-stream");+ if (!audioFile) return;+ app.commandLine.appendSwitch("use-fake-ui-for-media-stream");+ app.commandLine.appendSwitch("use-file-for-fake-audio-capture", `${audioFile}%noloop`);}function isLocalRendererOrigin(origin) {diff --git a/apps/desktop/electron/media-permissions.test.mjs b/apps/desktop/electron/media-permissions.test.mjsnew file mode 100644index 00000000..a3e05d46--- /dev/null+++ b/apps/desktop/electron/media-permissions.test.mjs@@ -0,0 +1,31 @@+import assert from "node:assert/strict";+import test from "node:test";++import { configureFakeMediaForTests } from "./media-permissions.mjs";++function fakeApp() {+ const switches = [];+ return { switches, commandLine: { appendSwitch: (...args) => switches.push(args) } };+}++test("no test switches without the flags", () => {+ const app = fakeApp();+ configureFakeMediaForTests(app, false, {});+ assert.deepEqual(app.switches, []);+});++test("OMNIRUSH_ELECTRON_FAKE_MEDIA gives a fake device only", () => {+ const app = fakeApp();+ configureFakeMediaForTests(app, true, {});+ assert.deepEqual(app.switches, [["use-fake-device-for-media-stream"]]);+});++test("OMNIRUSH_VOICE_INPUT_FILE plays the WAV once through the fake microphone", () => {+ const app = fakeApp();+ configureFakeMediaForTests(app, false, { OMNIRUSH_VOICE_INPUT_FILE: " /tmp/clip.wav " });+ assert.deepEqual(app.switches, [+ ["use-fake-device-for-media-stream"],+ ["use-fake-ui-for-media-stream"],+ ["use-file-for-fake-audio-capture", "/tmp/clip.wav%noloop"],+ ]);+});diff --git a/apps/desktop/package.json b/apps/desktop/package.jsonindex 715b3f03..21cc4b25 100644--- a/apps/desktop/package.json+++ b/apps/desktop/package.json@@ -27,7 +27,7 @@"prepare:sidecar": "node ./scripts/prepare-sidecar.mjs","pretest": "pnpm --filter @omnirush/headless-threads build","test:omnirush-account": "node --test electron/omnirush-account.test.mjs",- "test": "node --test scripts/prepare-server-constants.test.mjs scripts/verify-update-manifest.test.mjs electron/after-pack-paths.test.mjs electron/agent-context-diagnostics-fetch.test.mjs electron/automation-runner.test.mjs electron/binary-transfer.test.mjs electron/blank-slate-profile.test.mjs electron/blank-slate-profile-isolation.test.mjs electron/brand-app-name.test.mjs electron/brand-icon-darwin.test.mjs electron/brand-icon-windows.test.mjs electron/browser-login-sync.test.mjs electron/browser-panel.test.mjs electron/connect-link.test.mjs electron/desktop-distribution.test.mjs electron/dev-profile.test.mjs electron/electron-builder-config.test.mjs electron/linux-desktop-integration.test.mjs electron/linux-password-store.test.mjs electron/logs-folder.test.mjs electron/no-bare-external-fetch.test.mjs electron/nuke.test.mjs electron/omnirush-account.test.mjs electron/omnirush-ui-mcp.test.mjs electron/open-external.test.mjs electron/process-resilience.test.mjs electron/runtime.test.mjs electron/runtime-ca.test.mjs electron/runtime-chain-repair.test.mjs electron/sentry.test.mjs electron/server-dependency-mirror.test.mjs electron/secure-vault-key.test.mjs electron/stdio-errors.test.mjs electron/ui-control-server.test.mjs electron/updater.test.mjs electron/remote-workspace.test.mjs electron/workspace-archive.test.mjs electron/workspace-store.test.mjs",+ "test": "node --test scripts/prepare-server-constants.test.mjs scripts/verify-update-manifest.test.mjs electron/after-pack-paths.test.mjs electron/agent-context-diagnostics-fetch.test.mjs electron/automation-runner.test.mjs electron/binary-transfer.test.mjs electron/blank-slate-profile.test.mjs electron/blank-slate-profile-isolation.test.mjs electron/brand-app-name.test.mjs electron/brand-icon-darwin.test.mjs electron/brand-icon-windows.test.mjs electron/browser-login-sync.test.mjs electron/browser-panel.test.mjs electron/connect-link.test.mjs electron/desktop-distribution.test.mjs electron/dev-profile.test.mjs electron/electron-builder-config.test.mjs electron/linux-desktop-integration.test.mjs electron/linux-password-store.test.mjs electron/logs-folder.test.mjs electron/media-permissions.test.mjs electron/no-bare-external-fetch.test.mjs electron/nuke.test.mjs electron/omnirush-account.test.mjs electron/omnirush-ui-mcp.test.mjs electron/open-external.test.mjs electron/process-resilience.test.mjs electron/runtime.test.mjs electron/runtime-ca.test.mjs electron/runtime-chain-repair.test.mjs electron/sentry.test.mjs electron/server-dependency-mirror.test.mjs electron/secure-vault-key.test.mjs electron/stdio-errors.test.mjs electron/ui-control-server.test.mjs electron/updater.test.mjs electron/remote-workspace.test.mjs electron/workspace-archive.test.mjs electron/workspace-store.test.mjs","test:core": "node --test scripts/prepare-server-constants.test.mjs electron/workspace-store.test.mjs electron/workspace-archive.test.mjs electron/connect-link.test.mjs electron/open-external.test.mjs electron/secure-vault-key.test.mjs electron/process-resilience.test.mjs electron/stdio-errors.test.mjs electron/runtime-ca.test.mjs electron/runtime-chain-repair.test.mjs electron/automation-runner.test.mjs","pretest:core": "pnpm --filter @omnirush/headless-threads build"},