A desktop coding agent with free access to frontier models.
TypeScript
fix computer use setup persistence and loading (#67)
Slendywye2ff3b45395e
diff --git a/apps/app/src/app/constants.test.ts b/apps/app/src/app/constants.test.tsindex 76144cb5..6a5e47a3 100644--- a/apps/app/src/app/constants.test.ts+++ b/apps/app/src/app/constants.test.ts@@ -9,6 +9,7 @@ import {DEFAULT_MODEL,isOmniRushModelID,OMNIRUSH_EXTENSION_CATALOG,+ MCP_QUICK_CONNECT,filterOmniRushExtensionCatalogForPlatform,resolveOmniRushExtensionCatalogPlatform,} from "./constants";@@ -51,4 +52,9 @@ describe("omnirush.ai extension catalog platform filter", () => {expect(filteredIds("linux")).toEqual(["omnirush-browser", "computer-use", "ollama"]);expect(filteredIds("windows")).toEqual(["omnirush-browser", "computer-use", "ollama"]);});++ test("ships Computer Use without the preview catalog badge", () => {+ const computerUse = MCP_QUICK_CONNECT.find((entry) => entry.id === "computer-use");+ expect(computerUse?.extensionManifest?.preview).toEqual(undefined);+ });});diff --git a/apps/app/src/app/extensions.ts b/apps/app/src/app/extensions.tsindex 586c1de1..d095d5e1 100644--- a/apps/app/src/app/extensions.ts+++ b/apps/app/src/app/extensions.ts@@ -189,7 +189,6 @@ export const BUILT_IN_OMNIRUSH_EXTENSION_MANIFESTS: OmniRushExtensionManifest[]id: "computer-use",name: "Computer Use",description: "Work in the app and window you approve. Read, use accessible controls, or allow mouse and keyboard control with a small window preview.",- preview: true,source: { format: "omnirush-builtin", origin: "builtin", trusted: true },icon: { src: "/omnirush-mark.png" },composer: { prompt: "Use Computer Use to " },diff --git a/apps/app/src/react-app/domains/connections/store.ts b/apps/app/src/react-app/domains/connections/store.tsindex 37e2e7da..f58b888d 100644--- a/apps/app/src/react-app/domains/connections/store.ts+++ b/apps/app/src/react-app/domains/connections/store.ts@@ -75,6 +75,8 @@ const CLOUD_MCP_REFRESH_MARGIN_MS = 24 * 60 * 60 * 1000;const LOCAL_OMNIRUSH_SERVER_RECOVERY_TIMEOUT_MS = 30_000;const OMNIRUSH_UI_MCP_UNAVAILABLE = "UI control requires the omnirush.ai desktop app. Restart omnirush.ai or reinstall the app.";const COMPUTER_USE_HELPER_UNAVAILABLE = "Computer Use helper app is unavailable. Restart omnirush.ai or reinstall the app.";+const COMPUTER_USE_CONNECT_TIMEOUT_MS = 15_000;+const COMPUTER_USE_RECONNECT_COOLDOWN_MS = 30_000;async function withLocalOmniRushServerRecoveryTimeout<T>(task: Promise<T>,@@ -91,6 +93,34 @@ async function withLocalOmniRushServerRecoveryTimeout<T>(}}+function observedMcpStatus(value: unknown): string | null {+ if (!value || typeof value !== "object" || !("status" in value)) return null;+ const status = value.status;+ return typeof status === "string" ? status : null;+}++async function waitForMcpConnected(+ client: Client,+ name: string,+ directory: string,+): Promise<boolean> {+ const deadline = Date.now() + COMPUTER_USE_CONNECT_TIMEOUT_MS;+ while (Date.now() < deadline) {+ try {+ const response = unwrap(await client.mcp.status({ directory })) as Record<string, unknown>;+ const status = observedMcpStatus(response[name]);+ if (status === "connected") return true;+ if (status === "failed" || status === "needs_auth" || status === "needs_client_registration" || status === "disabled") {+ return false;+ }+ } catch {+ // The engine may not expose status until the helper transport starts.+ }+ await new Promise((resolve) => setTimeout(resolve, 250));+ }+ return false;+}+export type ConnectionsStoreSnapshot = {mcpServers: McpServerEntry[];mcpStatus: string | null;@@ -138,6 +168,8 @@ export function createConnectionsStore(options: {let snapshot: ConnectionsStoreSnapshot;const mcpStatusSynchronizer = createMcpStatusSynchronizer();const authStatusPolls = new Set<string>();+ const computerUseReconnectAt = new Map<string, number>();+ const computerUseReconnectInFlight = new Set<string>();let state: MutableState = {mcpServers: [],@@ -182,6 +214,43 @@ export function createConnectionsStore(options: {mutateState((current) => ({ ...current, [key]: value }));};+ const reconnectConfiguredComputerUse = (+ client: Client,+ directory: string,+ entries: McpServerEntry[],+ observedStatuses: Record<string, unknown>,+ ) => {+ if (!isDesktopRuntime() || options.workspaceType() !== "local" || !directory) return;+ const entry = entries.find((candidate) => candidate.name === "computer-use" || candidate.id === "computer-use");+ if (!entry || entry.config.type !== "local" || entry.config.enabled === false) return;+ if (state.mcpConnectingName) return;++ const status = observedMcpStatus(observedStatuses[entry.name] ?? observedStatuses[entry.id ?? ""]);+ if (status && status !== "disconnected" && status !== "failed") return;+ const key = `${directory}\u0000${entry.name}`;+ if (computerUseReconnectInFlight.has(key)) return;+ const lastAttemptAt = computerUseReconnectAt.get(key) ?? 0;+ if (Date.now() - lastAttemptAt < COMPUTER_USE_RECONNECT_COOLDOWN_MS) return;+ computerUseReconnectAt.set(key, Date.now());+ computerUseReconnectInFlight.add(key);+ const displayName = MCP_QUICK_CONNECT.find((candidate) => candidate.id === "computer-use")?.name ?? "Computer Use";+ mutateState((current) => ({ ...current, mcpConnectingName: displayName, mcpStatus: null }));++ void (async () => {+ try {+ await client.mcp.connect({ name: entry.name, directory });+ const connected = await waitForMcpConnected(client, entry.name, directory);+ if (!connected) setStateField("mcpStatus", "Computer Use is still starting. Refresh the status if it does not connect.");+ } catch (error) {+ setStateField("mcpStatus", error instanceof Error ? error.message : "Computer Use could not reconnect.");+ } finally {+ computerUseReconnectInFlight.delete(key);+ setStateField("mcpConnectingName", null);+ await refreshMcpServers();+ }+ })();+ };+const applyStateAction = <T,>(current: T, next: SetStateAction<T>) =>typeof next === "function" ? (next as (value: T) => T)(current) : next;@@ -402,10 +471,12 @@ export function createConnectionsStore(options: {const engineSync = response.engineSync ?? null;let nextStatuses: McpStatusMap = {};+ let observedStatuses: Record<string, unknown> = {};// Read through the same workspace mount as configuration. The chat client// can still point at the previous/default workspace during restoration.try {- nextStatuses = filterConfiguredStatuses(await omnirushClient.getMcpStatus(omnirushWorkspaceId), next);+ observedStatuses = await omnirushClient.getMcpStatus(omnirushWorkspaceId);+ nextStatuses = filterConfiguredStatuses(observedStatuses as McpStatusMap, next);} catch {nextStatuses = {};}@@ -434,6 +505,7 @@ export function createConnectionsStore(options: {return {next,nextStatuses,+ observedStatuses,engineSync,managedOAuthAvailable: response.managedOAuthState?.available ?? true,};@@ -562,6 +634,10 @@ export function createConnectionsStore(options: {? `Some MCPs could not be registered with the engine: ${failedNames}. They may appear disconnected — try reloading the engine.`: serverResult.next.length ? null : "No MCP servers configured yet.",}));+ const activeClient = options.client();+ if (activeClient && projectDir) {+ reconnectConfiguredComputerUse(activeClient, projectDir, serverResult.next, serverResult.observedStatuses);+ }void healUnhealthyMcpEntries(serverResult.next, projectedStatuses, refreshToken);return;}@@ -668,11 +744,12 @@ export function createConnectionsStore(options: {}let nextStatuses = state.mcpStatuses;+ let observedStatuses: Record<string, unknown> = {};const activeClient = options.client();if (activeClient) {try {- const status = unwrap(await activeClient.mcp.status({ directory: projectDir }));- nextStatuses = filterConfiguredStatuses(status as McpStatusMap, next);+ observedStatuses = unwrap(await activeClient.mcp.status({ directory: projectDir })) as Record<string, unknown>;+ nextStatuses = filterConfiguredStatuses(observedStatuses as McpStatusMap, next);} catch {nextStatuses = {};}@@ -687,6 +764,7 @@ export function createConnectionsStore(options: {mcpStatuses: projectedStatuses,mcpStatus: next.length ? null : "No MCP servers configured yet.",}));+ if (activeClient) reconnectConfiguredComputerUse(activeClient, projectDir, next, observedStatuses);void healUnhealthyMcpEntries(next, projectedStatuses, refreshToken);} catch (error) {if (!isCurrentRefresh()) return;@@ -1038,6 +1116,12 @@ export function createConnectionsStore(options: {config: mcpAddConfig,}),);+ if (entryType === "local" && slug === "computer-use") {+ await activeClient.mcp.connect({ name: slug, directory: resolvedProjectDir });+ if (!(await waitForMcpConnected(activeClient, slug, resolvedProjectDir))) {+ throw new Error("Computer Use is still starting. Refresh the status if it does not connect.");+ }+ }}options.markReloadRequired?.("mcp", { type: "mcp", name: slug, action });await refreshMcpServers();diff --git a/apps/app/src/react-app/domains/settings/computer-use-config.tsx b/apps/app/src/react-app/domains/settings/computer-use-config.tsxindex e4c4009e..b5429221 100644--- a/apps/app/src/react-app/domains/settings/computer-use-config.tsx+++ b/apps/app/src/react-app/domains/settings/computer-use-config.tsx@@ -1,5 +1,5 @@/** @jsxImportSource react */-import { useEffect } from "react";+import { useEffect, useRef, useState } from "react";import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";import { CheckCircle2, CircleAlert, Copy, Eye, Hand, Loader2, MousePointer2, RefreshCw, ShieldCheck } from "lucide-react";@@ -64,6 +64,12 @@ const modes = [export function ComputerUseConfig({ connected, connecting, onConnect, onRefresh, onPermissionsChange }: ComputerUseConfigProps) {const queryClient = useQueryClient();+ const [waitingForConnection, setWaitingForConnection] = useState(false);+ const connectedRef = useRef(connected);+ useEffect(() => {+ connectedRef.current = connected;+ if (connected) setWaitingForConnection(false);+ }, [connected]);const { data: result, isFetching, error: checkError, refetch } = useQuery({queryKey: PERMISSIONS_QUERY_KEY,queryFn: async () => {@@ -80,7 +86,20 @@ export function ComputerUseConfig({ connected, connecting, onConnect, onRefresh,refetchIntervalInBackground: true,});const connect = useMutation({- mutationFn: async () => { await onConnect?.(); await onRefresh?.(); },+ mutationFn: async () => {+ setWaitingForConnection(true);+ await onConnect?.();+ const deadline = Date.now() + 15_000;+ while (!connectedRef.current && Date.now() < deadline) {+ await onRefresh?.();+ if (connectedRef.current) break;+ await new Promise((resolve) => setTimeout(resolve, 250));+ }+ if (!connectedRef.current) {+ throw new Error("Computer Use is still starting. Refresh the status and try again if it does not connect.");+ }+ },+ onSettled: () => setWaitingForConnection(false),});const setup = useMutation({mutationFn: async () => parsePermissionResult(await desktopBridge.openComputerUsePermissionSetup()),@@ -103,7 +122,7 @@ export function ComputerUseConfig({ connected, connecting, onConnect, onRefresh,const supported = hasDesktopBridge() && result?.supported !== false;const permissionsReady = result?.ok === true;const ready = connected && permissionsReady;- const busy = isFetching || setup.isPending;+ const busy = isFetching || waitingForConnection || setup.isPending;const refresh = async () => { setup.reset(); connect.reset(); await refetch(); await onRefresh?.(); };return (@@ -125,9 +144,9 @@ export function ComputerUseConfig({ connected, connecting, onConnect, onRefresh,<span>{ready ? "Ready · app access is approved when a session starts" : !supported ? "Desktop access is unavailable on this session" : !connected ? permissionsReady ? "Permissions are ready. Enable Computer Use for this workspace." : "Set up Computer Use for this workspace" : mac ? "Connected · finish macOS permissions below" : "Connected · check desktop access below"}</span></div>{!ready ? (- <Button className="w-full" onClick={() => { if (!connected) connect.mutate(); else setup.mutate(); }} disabled={!supported || connecting || connect.isPending || setup.isPending || (!connected && !onConnect)}>- {connecting || connect.isPending || setup.isPending ? <Loader2 className="size-4 animate-spin" /> : null}- {connecting || connect.isPending ? "Enabling…" : !connected ? "Enable Computer Use" : mac ? "Allow macOS access" : "Check desktop access"}+ <Button className="w-full" onClick={() => { if (!connected) connect.mutate(); else setup.mutate(); }} disabled={!supported || connecting || waitingForConnection || connect.isPending || setup.isPending || (!connected && !onConnect)}>+ {connecting || waitingForConnection || connect.isPending || setup.isPending ? <Loader2 className="size-4 animate-spin" /> : null}+ {connecting || waitingForConnection || connect.isPending ? "Enabling…" : !connected ? "Enable Computer Use" : mac ? "Allow macOS access" : "Check desktop access"}</Button>) : null}{error ? <Alert variant="destructive"><CircleAlert /><AlertDescription className="break-words">{error}</AlertDescription></Alert> : null}diff --git a/apps/app/src/react-app/domains/settings/settings-extension-controller.ts b/apps/app/src/react-app/domains/settings/settings-extension-controller.tsindex 31e7da7b..155317a7 100644--- a/apps/app/src/react-app/domains/settings/settings-extension-controller.ts+++ b/apps/app/src/react-app/domains/settings/settings-extension-controller.ts@@ -56,7 +56,7 @@ export function useSettingsExtensionController(input: SettingsExtensionControllerestartLocalServer: input.restartLocalServer,computerUse: {connected: input.enablementContext.mcpStatuses?.["computer-use"]?.status === "connected",- connecting: input.mcpConnectingName === entry.name,+ connecting: input.mcpConnectingName === entry.name || input.mcpConnectingName === entry.serverName,onConnect: () => input.connectMcp(entry),onRefresh: input.refreshMcpServers,onPermissionsChange: input.onComputerUsePermissionsChange,diff --git a/packages/docs/start-here/do-work-with-it/use-desktop-apps.mdx b/packages/docs/start-here/do-work-with-it/use-desktop-apps.mdxindex eb8f08cc..a704b926 100644--- a/packages/docs/start-here/do-work-with-it/use-desktop-apps.mdx+++ b/packages/docs/start-here/do-work-with-it/use-desktop-apps.mdx@@ -1,19 +1,21 @@---title: "Work in desktop apps"-description: "Approve a Mac app window and choose how OmniRush.ai can help"+description: "Approve an app window and choose how OmniRush.ai can help"----Computer Use is a preview for macOS 14 or later. It can read a window, use-accessible app controls, or work with mouse and keyboard in the foreground.-Use the built-in browser for websites and a dedicated connection when an app-provides one.+Computer Use can read a window, use accessible app controls, or work with mouse+and keyboard in the foreground. It is available on macOS 14+, Windows, and+Linux X11. Use the built-in browser for websites and a dedicated connection+when an app provides one.## Set up-1. Open **Settings → Library → Computer Use** and enable it.+1. Open **Settings → Library → Computer Use** and enable it. OmniRush remembers+ this for the workspace and reconnects it when the desktop app starts.2. Open **Permissions and session controls**.-3. Grant **Accessibility** and **Screen Recording** to **OmniRush.ai Computer Use**- in macOS settings. OmniRush.ai cannot grant these permissions for you.+3. On macOS, grant **Accessibility** and **Screen Recording** to **OmniRush.ai+ Computer Use** in macOS settings. OmniRush.ai cannot grant these permissions+ for you. Windows and Linux require the target app to remain in the foreground.4. Return to OmniRush.ai and check readiness. Reconnect Computer Use if macOS asksyou to restart the helper or if you are upgrading from the previous version.5. Open the app you want to use and mention it in your message.@@ -65,7 +67,8 @@ OmniRush.ai itself. App access can still allow meaningful changes within thatapp, including saved files or sent messages, so describe your intended taskclearly and remain present for sensitive work.-Desktop control on Windows and Linux, locked-desktop use, background mouse and-keyboard input, arbitrary keyboard shortcuts, and persistent app approvals are-not available in this preview. The OmniRush.ai Browser remains available for-website tasks on supported desktop platforms.+Desktop control on Windows and Linux requires the target app to remain in the+foreground. Locked-desktop use, background mouse and keyboard input, arbitrary+keyboard shortcuts, and persistent app approvals are not available. The+OmniRush.ai Browser remains available for website tasks on supported desktop+platforms.