Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

feat: add approved Windows and Linux X11 computer use

Task Builder90a0523c2cc1
diff --git a/.github/workflows/computer-use-native.yml b/.github/workflows/computer-use-native.yml
new file mode 100644
index 00000000..2a115628
--- /dev/null
+++ b/.github/workflows/computer-use-native.yml
@@ -0,0 +1,61 @@
+name: Native Computer Use
+on:
+ pull_request:
+ paths:
+ - "apps/desktop/electron/computer-use*"
+ - "apps/desktop/electron-builder.base.yml"
+ - "apps/desktop/package.json"
+ - "apps/desktop/scripts/prepare-runtime-node-modules.mjs"
+ - "apps/app/src/**/computer-use*"
+ - "evals/worlds/computer-use.ts"
+ - "evals/packages/labs/fixtures/portable-computer-use*"
+ - "evals/specs/computer-use-portable-window.e2e.test.ts"
+ - ".github/workflows/computer-use-native.yml"
+ workflow_dispatch:
+permissions:
+ contents: read
+jobs:
+ native-window-journey:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [windows-latest, ubuntu-latest]
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.sha }}
+ - uses: actions/setup-node@v6
+ with:
+ node-version: 24
+ - uses: pnpm/action-setup@v4
+ with:
+ version: 11.4.0
+ - name: Install app and journey dependencies
+ shell: bash
+ run: |
+ pnpm install --frozen-lockfile
+ pnpm --dir evals install --frozen-lockfile
+ - name: Prepare an isolated Linux desktop
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get update -qq
+ sudo apt-get install -y xvfb openbox libxi6 libxtst6
+ - name: Run Windows native journey
+ if: runner.os == 'Windows'
+ shell: bash
+ run: pnpm evals:e2e computer-use-portable-window
+ - name: Run Linux X11 native journey
+ if: runner.os == 'Linux'
+ run: |
+ xvfb-run -a sh -c 'openbox --sm-disable >/tmp/omnirush-test-openbox.log 2>&1 & export XDG_SESSION_TYPE=x11; unset WAYLAND_DISPLAY; pnpm evals:e2e computer-use-portable-window'
+ - name: Preserve journey evidence
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: computer-use-${{ runner.os }}-${{ github.sha }}
+ path: |
+ evals/results/test-runs/**
+ evals/results/.testkit/**
+ retention-days: 14
diff --git a/apps/app/src/app/constants.test.ts b/apps/app/src/app/constants.test.ts
index 2062394a..76144cb5 100644
--- a/apps/app/src/app/constants.test.ts
+++ b/apps/app/src/app/constants.test.ts
@@ -46,8 +46,9 @@ describe("omnirush.ai extension catalog platform filter", () => {
expect(filteredIds("web")).toEqual(["ollama"]);
});
- test("keeps omnirush.ai Browser desktop-only and Computer Use mac-only", () => {
+ test("keeps Browser and Computer Use on desktop platforms", () => {
expect(filteredIds("darwin")).toEqual(["omnirush-browser", "computer-use", "ollama"]);
- expect(filteredIds("linux")).toEqual(["omnirush-browser", "ollama"]);
+ expect(filteredIds("linux")).toEqual(["omnirush-browser", "computer-use", "ollama"]);
+ expect(filteredIds("windows")).toEqual(["omnirush-browser", "computer-use", "ollama"]);
});
});
diff --git a/apps/app/src/app/extensions.ts b/apps/app/src/app/extensions.ts
index ee4de7ac..586c1de1 100644
--- a/apps/app/src/app/extensions.ts
+++ b/apps/app/src/app/extensions.ts
@@ -188,15 +188,15 @@ export const BUILT_IN_OMNIRUSH_EXTENSION_MANIFESTS: OmniRushExtensionManifest[]
schemaVersion: 1,
id: "computer-use",
name: "Computer Use",
- description: "Work in the Mac app and window you approve. Read, use accessible controls, or allow mouse and keyboard control with a small window preview.",
+ description: "Work in the app and window you approve. Read, use accessible controls, or allow mouse and keyboard control with a small window preview.",
preview: true,
source: { format: "omnirush-builtin", origin: "builtin", trusted: true },
icon: { src: "/omnirush-mark.png" },
composer: { prompt: "Use Computer Use to " },
setup: {
- instructions: "Computer Use is available on macOS 14 or later. Grant Accessibility and Screen Recording in the helper. For each session, choose an app window and allow reading, app controls, or mouse and keyboard. Choose Allow and start in omnirush.ai. Your input interrupts control; Stop in the preview ends access.",
+ instructions: "Computer Use supports macOS 14+, Windows, and Linux X11. On macOS, grant Accessibility and Screen Recording. Windows and Linux control require the app in front; your input pauses control. Wayland requires a separate desktop portal integration. For each session, choose an app window and allow reading, app controls, or mouse and keyboard. Choose Allow and start in omnirush.ai. Your input interrupts control; Stop in the preview ends access.",
primaryCta: "Enable Computer Use",
- secondaryCta: "Check macOS permissions",
+ secondaryCta: "Check desktop access",
testActionRef: "omnirush.computerUse.healthCheck",
},
resources: [
@@ -230,7 +230,7 @@ export const BUILT_IN_OMNIRUSH_EXTENSION_MANIFESTS: OmniRushExtensionManifest[]
],
lifecycle: { reload: ["mcp"], detection: ["mcp:computer-use"] },
defaultEnabled: true,
- platform: ["darwin"],
+ platform: ["darwin", "windows", "linux"],
},
{
schemaVersion: 1,
diff --git a/apps/app/src/react-app/domains/connections/store.ts b/apps/app/src/react-app/domains/connections/store.ts
index 846b7f1b..37e2e7da 100644
--- a/apps/app/src/react-app/domains/connections/store.ts
+++ b/apps/app/src/react-app/domains/connections/store.ts
@@ -464,7 +464,7 @@ export function createConnectionsStore(options: {
const mcpResource = extensionResource(entry.extensionManifest, "mcp");
if (mcpResource?.localCommandRef === "omnirush.computerUseMcp") {
const command = await resolveDesktopCommand("getComputerUseMcpCommand", COMPUTER_USE_HELPER_UNAVAILABLE);
- if (!command) throw new Error("Computer Use requires the bundled omnirush.ai helper on macOS.");
+ if (!command) throw new Error("Computer Use is unavailable on this desktop. Check its setup page.");
return command;
}
if (isOmniRushUiMcpEntry(entry)) {
@@ -476,6 +476,11 @@ export function createConnectionsStore(options: {
};
const resolveLocalMcpEnvironment = async (entry: McpDirectoryInfo) => {
+ if (extensionResource(entry.extensionManifest, "mcp")?.localCommandRef === "omnirush.computerUseMcp") {
+ const environment = await window.__OMNIRUSH_ELECTRON__?.invokeDesktop?.("getComputerUseMcpEnvironment");
+ if (!environment) throw new Error(COMPUTER_USE_HELPER_UNAVAILABLE);
+ return environment;
+ }
if (!isOmniRushUiMcpEntry(entry)) return undefined;
// Required, not best-effort: packaged launches run the app's own binary and
// need ELECTRON_RUN_AS_NODE=1 from this environment to act as Node.
diff --git a/apps/app/src/react-app/domains/session/surface/composer/app-mentions.ts b/apps/app/src/react-app/domains/session/surface/composer/app-mentions.ts
index fdedee22..aaa11dbd 100644
--- a/apps/app/src/react-app/domains/session/surface/composer/app-mentions.ts
+++ b/apps/app/src/react-app/domains/session/surface/composer/app-mentions.ts
@@ -1,17 +1,15 @@
import { OMNIRUSH_EXTENSION_CATALOG } from "@/app/constants";
import { desktopBridge } from "@/app/lib/desktop";
-import { isMacPlatform } from "@/app/utils";
import { isOmniRushExtensionEnabled, isOmniRushExtensionHidden } from "@/react-app/domains/settings/extension-state";
/**
- * "@App" mentions let the user target a running macOS app for Computer Use
+ * "@App" mentions let the user target a running desktop app for Computer Use
* straight from the composer (e.g. "@Music"). They are only offered when:
- * - running inside the Electron desktop shell on macOS, and
- * - the Computer Use extension is enabled (it is macOS-only and opt-in).
+ * - running inside the Electron desktop shell, and
+ * - the Computer Use extension is enabled (it is enabled for this workspace).
*/
export function isAppMentionAvailable(): boolean {
if (typeof window === "undefined" || !window.__OMNIRUSH_ELECTRON__?.invokeDesktop) return false;
- if (!isMacPlatform()) return false;
const entry = OMNIRUSH_EXTENSION_CATALOG.find((candidate) => candidate.id === "computer-use");
if (!entry) return false;
return isOmniRushExtensionEnabled(entry) && !isOmniRushExtensionHidden(entry);
@@ -23,7 +21,7 @@ export function isAppMentionAvailable(): boolean {
* app instead of assuming the app is frontmost.
*/
export function appMentionInstruction(appName: string) {
- return `[The user mentioned the macOS app ${JSON.stringify(appName)}. Prefer a dedicated app integration when available. For Computer Use, start with computer_discover to resolve its exact app_id, then computer_open_session with the least powerful mode needed (observe, assist, or control). The person approves the app and chooses a window. Use computer_observe before acting and after each computer_act. Use observed element refs before visual coordinates. A mention is a target, not a permission grant; respect denial, pause and human takeover.]`;
+ return `[The user mentioned the desktop app ${JSON.stringify(appName)}. Prefer a dedicated app integration when available. For Computer Use, start with computer_discover to resolve its exact app_id, then computer_open_session with the least powerful mode the discovered platform supports. The person approves the app and chooses a window. Use computer_observe before acting and after each computer_act. Use observed element refs when available; otherwise use the screenshot coordinates returned by the tool. A mention is a target, not a permission grant; respect denial, pause and human takeover.]`;
}
type ListRunningAppsResult = { ok?: boolean; apps?: unknown };
@@ -31,7 +29,7 @@ type ListRunningAppsResult = { ok?: boolean; apps?: unknown };
let appsCache: { at: number; apps: string[] } | null = null;
const APPS_CACHE_TTL_MS = 10_000;
-/** List running regular macOS apps for the mention menu. Cached briefly; never throws. */
+/** List running desktop apps for the mention menu. Cached briefly; never throws. */
export async function listRunningAppsForMention(): Promise<string[]> {
if (!isAppMentionAvailable()) return [];
if (appsCache && Date.now() - appsCache.at < APPS_CACHE_TTL_MS) return appsCache.apps;
diff --git a/apps/app/src/react-app/domains/session/surface/computer-use-controls.tsx b/apps/app/src/react-app/domains/session/surface/computer-use-controls.tsx
index 064ae6e1..24667941 100644
--- a/apps/app/src/react-app/domains/session/surface/computer-use-controls.tsx
+++ b/apps/app/src/react-app/domains/session/surface/computer-use-controls.tsx
@@ -42,7 +42,7 @@ export function ComputerUseControls() {
{session.windows?.map((window) => <option key={window.id} value={window.id}>{window.title}</option>)}
</select>
</label>
- <p className="my-3 text-xs text-muted-foreground">{session.mode === "observe" ? "Read this window. No clicks or typing." : session.mode === "assist" ? "Read and use this window’s app controls." : "Use this window’s mouse and keyboard. omnirush.ai yields to your input, then refreshes the window before continuing. Stop ends access."} Access lasts up to 15 minutes. Window content goes to your selected model provider.</p>
+ <p className="my-3 text-xs text-muted-foreground">{session.mode === "observe" ? "Read this window. No clicks or typing." : session.mode === "assist" ? "Read and use this window’s app controls." : "Use this window’s mouse and keyboard. Your input interrupts control. Stop ends access."} Access lasts up to 15 minutes. Window content goes to your selected model provider.</p>
<div className="flex justify-end gap-2"><Button variant="outline" onClick={() => send("deny")}>Cancel</Button><Button disabled={selectedWindow === undefined || action.isPending} onClick={() => send("approve")}>Allow and start</Button></div>
</> : <>
<p className="mt-2 truncate text-xs text-muted-foreground">{session.windowTitle} · {session.mode === "observe" ? "Read only" : session.mode === "assist" ? "App controls" : "Mouse and keyboard"}</p>
diff --git a/apps/app/src/react-app/domains/settings/computer-use-config.tsx b/apps/app/src/react-app/domains/settings/computer-use-config.tsx
index cb990745..44c50cee 100644
--- a/apps/app/src/react-app/domains/settings/computer-use-config.tsx
+++ b/apps/app/src/react-app/domains/settings/computer-use-config.tsx
@@ -1,7 +1,7 @@
/** @jsxImportSource react */
import { useEffect } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
-import { CheckCircle2, CircleAlert, Eye, Hand, Loader2, MousePointer2, RefreshCw, ShieldCheck } from "lucide-react";
+import { CheckCircle2, CircleAlert, Copy, Eye, Hand, Loader2, MousePointer2, RefreshCw, ShieldCheck } from "lucide-react";
import { desktopBridge } from "@/app/lib/desktop";
import { Alert, AlertDescription } from "@/components/ui/alert";
@@ -16,6 +16,8 @@ type PermissionResult = {
screenRecording: boolean;
supported: boolean;
error?: string;
+ platform?: string;
+ modes?: string[];
};
type ComputerUseConfigProps = {
@@ -47,15 +49,17 @@ function parsePermissionResult(value: unknown): PermissionResult {
accessibility: "accessibility" in value && value.accessibility === true,
screenRecording: "screenRecording" in value && value.screenRecording === true,
supported: !("supported" in value && value.supported === false),
+ platform: "platform" in value && typeof value.platform === "string" ? value.platform : "darwin",
+ modes: "modes" in value && Array.isArray(value.modes) ? value.modes.filter((mode): mode is string => typeof mode === "string") : ["observe", "assist", "control"],
error: "error" in value && typeof value.error === "string" ? value.error : undefined,
};
}
const PERMISSIONS_QUERY_KEY = ["computer-use", "permissions"];
const modes = [
- { icon: Eye, name: "Read a window", description: "See its text and screenshots. No clicks or typing." },
- { icon: Hand, name: "Use app controls", description: "Work through accessible controls while your pointer stays free." },
- { icon: MousePointer2, name: "Use mouse and keyboard", description: "Work in the foreground. Your input pauses the session." },
+ { mode: "observe", icon: Eye, name: "Read a window", description: "See its text and screenshots. No clicks or typing." },
+ { mode: "assist", icon: Hand, name: "Use app controls", description: "Work through accessible controls while your pointer stays free." },
+ { mode: "control", icon: MousePointer2, name: "Use mouse and keyboard", description: "Work in the foreground. Your input pauses the session." },
];
export function ComputerUseConfig({ connected, connecting, onConnect, onRefresh, onPermissionsChange }: ComputerUseConfigProps) {
@@ -82,11 +86,20 @@ export function ComputerUseConfig({ connected, connecting, onConnect, onRefresh,
mutationFn: async () => parsePermissionResult(await desktopBridge.openComputerUsePermissionSetup()),
onSuccess: (next) => queryClient.setQueryData(PERMISSIONS_QUERY_KEY, next),
});
+ const copyCli = useMutation({
+ mutationFn: async () => {
+ const [command, env] = await Promise.all([desktopBridge.getComputerUseMcpCommand(), desktopBridge.getComputerUseMcpEnvironment()]);
+ if (!command.length || !command.every((part) => typeof part === "string") || !env) throw new Error("Computer Use could not create a CLI connection. Check desktop access and try again.");
+ await navigator.clipboard.writeText(JSON.stringify({ mcpServers: { "computer-use": { command: command[0], args: command.slice(1), env } } }, null, 2));
+ },
+ });
useEffect(() => {
if (result) onPermissionsChange?.({ accessibility: result.accessibility, screenRecording: result.screenRecording });
}, [result, onPermissionsChange]);
- const error = (connect.error ?? setup.error ?? checkError)?.message ?? result?.error;
+ const mac = result?.platform === "darwin";
+ const availableModes = result?.modes ?? ["observe", "control"];
+ const error = (connect.error ?? setup.error ?? copyCli.error ?? checkError)?.message ?? result?.error;
const supported = hasDesktopBridge() && result?.supported !== false;
const permissionsReady = result?.ok === true;
const ready = connected && permissionsReady;
@@ -98,7 +111,7 @@ export function ComputerUseConfig({ connected, connecting, onConnect, onRefresh,
<CardHeader>
<CardTitle>Work in an app you choose</CardTitle>
<CardDescription>
- Approve one Mac app, choose its window, and decide how omnirush.ai can help. Your input interrupts control; Stop in the preview ends access.
+ Approve one app, choose its window, and decide how omnirush.ai can help. Your input interrupts control; Stop in the preview ends access.
</CardDescription>
<CardAction>
<Button variant="ghost" size="icon-sm" aria-label="Refresh Computer Use status" onClick={() => void refresh()} disabled={busy}>
@@ -109,19 +122,19 @@ export function ComputerUseConfig({ connected, connecting, onConnect, onRefresh,
<CardContent className="space-y-5">
<div className="flex items-center gap-2 rounded-xl border border-border bg-muted/40 px-3 py-3 text-sm" role="status" aria-live="polite">
{ready ? <CheckCircle2 className="size-4 shrink-0 text-green-11" /> : <ShieldCheck className="size-4 shrink-0 text-muted-foreground" />}
- <span>{ready ? "Ready · app access is approved when a session starts" : !supported ? "Available in omnirush.ai for macOS 14 or later" : !connected ? permissionsReady ? "Permissions are ready. Enable Computer Use for this workspace." : "Set up Computer Use for this workspace" : "Connected · finish macOS permissions below"}</span>
+ <span>{ready ? "Ready · app access is approved when a session starts" : !supported ? "Desktop access is unavailable on this session" : !connected ? permissionsReady ? "Permissions are ready. Enable Computer Use for this workspace." : "Set up Computer Use for this workspace" : mac ? "Connected · finish macOS permissions below" : "Connected · check desktop access below"}</span>
</div>
{!ready ? (
<Button className="w-full" onClick={() => { if (!connected) connect.mutate(); else setup.mutate(); }} disabled={!supported || connecting || connect.isPending || setup.isPending || (!connected && !onConnect)}>
{connecting || connect.isPending || setup.isPending ? <Loader2 className="size-4 animate-spin" /> : null}
- {connecting || connect.isPending ? "Enabling…" : !connected ? "Enable Computer Use" : "Allow macOS access"}
+ {connecting || connect.isPending ? "Enabling…" : !connected ? "Enable Computer Use" : mac ? "Allow macOS access" : "Check desktop access"}
</Button>
) : null}
{error ? <Alert variant="destructive"><CircleAlert /><AlertDescription className="break-words">{error}</AlertDescription></Alert> : null}
<div className="space-y-3">
<p className="text-sm font-medium">You choose the scope</p>
<div className="grid gap-2">
- {modes.map(({ icon: Icon, name, description }) => (
+ {modes.filter(({ mode }) => availableModes.includes(mode)).map(({ icon: Icon, name, description }) => (
<div key={name} className="flex gap-3 rounded-xl border border-border p-3">
<Icon className="mt-0.5 size-4 shrink-0 text-muted-foreground" />
<div><p className="text-sm font-medium">{name}</p><p className="mt-1 text-xs leading-relaxed text-muted-foreground">{description}</p></div>
@@ -130,20 +143,25 @@ export function ComputerUseConfig({ connected, connecting, onConnect, onRefresh,
</div>
</div>
<div className="space-y-3 border-t border-border pt-4">
- <div><p className="text-sm font-medium">macOS permissions</p><p className="mt-1 text-xs leading-relaxed text-muted-foreground">These let the helper work. App access still needs your approval for each session.</p></div>
- <PermissionRow title="Accessibility" description="Read and use app controls" granted={result?.accessibility} />
- <PermissionRow title="Screen Recording" description="See the window you selected" granted={result?.screenRecording} />
+ <div><p className="text-sm font-medium">{mac ? "macOS permissions" : "Desktop access"}</p><p className="mt-1 text-xs leading-relaxed text-muted-foreground">These let the helper work. App access still needs your approval for each session.</p></div>
+ {mac ? <PermissionRow title="Accessibility" description="Read and use app controls" granted={result?.accessibility} /> : <PermissionRow title="Mouse and keyboard" description="Foreground input; your input pauses control" granted={result?.accessibility} />}
+ <PermissionRow title={mac ? "Screen Recording" : "Window screenshots"} description="See only the window you approve" granted={result?.screenRecording} />
<Button variant="outline" className="min-h-10 w-full whitespace-normal" disabled={!supported || setup.isPending} onClick={() => setup.mutate()}>
{setup.isPending ? <Loader2 className="size-4 animate-spin" /> : null}
Open permissions and session controls
</Button>
</div>
+ {permissionsReady ? <div className="space-y-2 border-t border-border pt-4">
+ <Button variant="outline" disabled={copyCli.isPending} onClick={() => copyCli.mutate()}><Copy className="size-4" />{copyCli.isSuccess ? "CLI connection copied" : "Copy CLI connection"}</Button>
+ <p className="text-xs leading-relaxed text-muted-foreground">Keep OmniRush.ai open. Add the copied server to ~/.omnirush/mcp.json, then run /mcp reconnect in the CLI. Window approval stays in this app.</p>
+ </div> : null}
<p className="text-xs leading-relaxed text-muted-foreground">
+ {!mac ? "Keep the target app visible on an unlocked desktop. Computer Use moves the mouse and types in the foreground; choose Continue after taking over. Choose a model that accepts images. For background Blender work, ask for a Blender Python script or use an app connector. " : null}
Sessions end after 15 minutes and pause when idle. Window content is processed by your selected model provider. Enter passwords yourself. For websites, use the built-in browser.
</p>
</CardContent>
<CardFooter className="flex flex-wrap items-center justify-between gap-3 border-t border-border">
- <p className="max-w-sm text-xs text-muted-foreground">{ready ? "Mention an app in your next message, then approve the window when asked." : !connected ? "macOS access and enabling tools for this workspace are separate steps." : "Allow access, then return here. Status updates automatically."}</p>
+ <p className="max-w-sm text-xs text-muted-foreground">{ready ? "Mention an app in your next message, then approve the window when asked." : !connected ? "Desktop access and enabling tools for this workspace are separate steps." : "Allow access, then return here. Status updates automatically."}</p>
{ready ? <Button variant="outline" onClick={() => void refresh()} disabled={busy}>Check readiness</Button> : null}
</CardFooter>
</Card>
diff --git a/apps/app/tests/mcp-local-server-recovery.test.ts b/apps/app/tests/mcp-local-server-recovery.test.ts
index 4bae8457..17ad0d99 100644
--- a/apps/app/tests/mcp-local-server-recovery.test.ts
+++ b/apps/app/tests/mcp-local-server-recovery.test.ts
@@ -79,7 +79,7 @@ describe("bundled Computer Use setup", () => {
async function connectWithHelper(command: string[] | null) {
Object.defineProperty(globalThis, "window", {
configurable: true,
- value: { __OMNIRUSH_ELECTRON__: { invokeDesktop: async (name: string) => name === "getComputerUseMcpCommand" ? command : null } },
+ value: { __OMNIRUSH_ELECTRON__: { invokeDesktop: async (name: string) => name === "getComputerUseMcpCommand" ? command : name === "getComputerUseMcpEnvironment" ? {} : null } },
});
const server = createOmniRushServerStore({
startupPreference: () => "server", documentVisible: () => true, developerMode: () => false,
@@ -110,12 +110,12 @@ describe("bundled Computer Use setup", () => {
const command = ["/Applications/omnirush.ai.app/Contents/Resources/helpers/omnirush.ai Computer Use.app/Contents/MacOS/ComputerUse", "mcp"];
const { result, saved } = await connectWithHelper(command);
expect(result).toEqual({ ok: true });
- expect(saved).toEqual([{ name: "computer-use", config: { type: "local", enabled: true, command } }]);
+ expect(saved).toEqual([{ name: "computer-use", config: { type: "local", enabled: true, command, environment: {} } }]);
});
test("missing helper returns an actionable error without saving a connection", async () => {
const { result, saved } = await connectWithHelper(null);
- expect(result).toEqual({ ok: false, error: "Computer Use requires the bundled omnirush.ai helper on macOS." });
+ expect(result).toEqual({ ok: false, error: "Computer Use is unavailable on this desktop. Check its setup page." });
expect(saved).toEqual([]);
});
});
diff --git a/apps/desktop/electron-builder.base.yml b/apps/desktop/electron-builder.base.yml
index 82beff6e..b54f6861 100644
--- a/apps/desktop/electron-builder.base.yml
+++ b/apps/desktop/electron-builder.base.yml
@@ -10,6 +10,8 @@ files:
- from: .electron-runtime/node_modules
to: node_modules
extraResources:
+ - from: electron/computer-use-relay.mjs
+ to: computer-use/relay.mjs
- from: .electron-runtime/omnirush-sentry.json
to: omnirush-sentry.json
# Bundled UI-control MCP (scripts/prepare-ui-mcp.mjs). Launched with the app's
@@ -33,6 +35,7 @@ extraResources:
- "docs.json"
asar: true
asarUnpack:
+ - node_modules/koffi/**
- node_modules/.pnpm/@lydell+node-pty*/**
- node_modules/node-pty/**
npmRebuild: false
diff --git a/apps/desktop/electron/computer-use-native-worker.mjs b/apps/desktop/electron/computer-use-native-worker.mjs
new file mode 100644
index 00000000..ccd6d35b
--- /dev/null
+++ b/apps/desktop/electron/computer-use-native-worker.mjs
@@ -0,0 +1,435 @@
+// Native desktop access lives off the Electron main thread. No shell execution.
+const parentPort = { postMessage: (value) => process.send?.(value), on: (name, listener) => process.on(name === "close" ? "disconnect" : name, listener), close: () => process.disconnect() };
+import { readFileSync, readlinkSync } from "node:fs";
+import path from "node:path";
+import koffi from "koffi";
+import { crc32, deflateSync } from "node:zlib";
+
+let generation = 0;
+let backend;
+const changed = () => { generation++; parentPort.postMessage({ event: "person-input", generation }); };
+const fail = (code, message) => { throw Object.assign(new Error(message), { code }); };
+const protectedNames = new Set(["cmd.exe", "powershell.exe", "pwsh.exe", "windowsterminal.exe", "conhost.exe", "credentialuibroker.exe", "consent.exe", "logonui.exe", "keepass.exe", "keepassxc.exe", "1password.exe", "gnome-terminal-server", "konsole", "xterm", "alacritty", "kitty", "foot", "gnome-keyring-daemon", "keepassxc", "1password"]);
+function safeWindow(value) {
+ if ((value.pid === process.pid || value.pid === process.ppid) || protectedNames.has(path.basename(value.executable).toLowerCase())) fail("protected_app", "Use a shell tool for terminals. Computer Use cannot operate the host, security prompts, or password managers.");
+ if (!Number.isSafeInteger(value.id) || !Number.isSafeInteger(value.pid) || value.pid <= 0) fail("window_unavailable", "This window cannot be identified safely.");
+ return value;
+}
+const keyNames = { Enter: 13, Tab: 9, Escape: 27, Backspace: 8, Delete: 46, Space: 32, Left: 37, Up: 38, Right: 39, Down: 40, Home: 36, End: 35, PageUp: 33, PageDown: 34 };
+function keys(action) {
+ if (typeof action.key !== "string") fail("invalid_action", "A key name is required.");
+ const modifiers = action.modifiers ?? [];
+ if (!Array.isArray(modifiers) || modifiers.some((v) => !["ctrl", "shift", "alt"].includes(v)) || new Set(modifiers).size !== modifiers.length) fail("invalid_action", "Only ctrl, shift, and alt modifiers are supported.");
+ if ((modifiers.includes("alt") && ["Tab", "Escape", "F4"].includes(action.key)) || (modifiers.includes("ctrl") && (modifiers.includes("alt") || action.key === "Escape"))) fail("unsupported_action", "System-wide shortcuts are unavailable.");
+ if (!(action.key in keyNames) && !/^[a-z0-9]$/i.test(action.key) && !/^F([1-9]|1[0-2])$/.test(action.key)) fail("unsupported_action", "This key is not supported.");
+ return modifiers;
+}
+
+function windows() {
+ const u = koffi.load("user32.dll"), k = koffi.load("kernel32.dll"), dwm = koffi.load("dwmapi.dll");
+ const f = (lib, declaration) => lib.func(declaration);
+ const rect = koffi.struct("CU_RECT", { left: "int32_t", top: "int32_t", right: "int32_t", bottom: "int32_t" });
+ const point = koffi.struct("CU_POINT", { x: "int32_t", y: "int32_t" });
+ const filetime = koffi.struct("CU_FILETIME", { low: "uint32_t", high: "uint32_t" });
+ const mouse = koffi.struct("CU_MOUSEINPUT", { dx: "int32_t", dy: "int32_t", mouseData: "uint32_t", dwFlags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t" });
+ const keyboard = koffi.struct("CU_KEYBDINPUT", { wVk: "uint16_t", wScan: "uint16_t", dwFlags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t" });
+ const hardware = koffi.struct("CU_HARDWAREINPUT", { uMsg: "uint32_t", wParamL: "uint16_t", wParamH: "uint16_t" });
+ const input = koffi.struct("CU_INPUT", { type: "uint32_t", value: koffi.union("CU_INPUT_VALUE", { mi: mouse, ki: keyboard, hi: hardware }) });
+ const hookKey = koffi.struct("CU_KBDLLHOOKSTRUCT", { vkCode: "uint32_t", scanCode: "uint32_t", flags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t" });
+ const hookMouse = koffi.struct("CU_MSLLHOOKSTRUCT", { pt: point, mouseData: "uint32_t", flags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t" });
+ const msg = koffi.struct("CU_MSG", { hwnd: "uintptr_t", message: "uint32_t", wParam: "uintptr_t", lParam: "intptr_t", time: "uint32_t", pt: point, lPrivate: "uint32_t" });
+ const enumProto = koffi.proto("int __stdcall CU_ENUMPROC(uintptr_t hwnd, intptr_t parameter)");
+ const hookProto = koffi.proto("intptr_t __stdcall CU_HOOKPROC(int code, uintptr_t wParam, intptr_t lParam)");
+ const enumWindows = f(u, "int __stdcall EnumWindows(CU_ENUMPROC *callback, intptr_t parameter)");
+ const visible = f(u, "int __stdcall IsWindowVisible(uintptr_t hwnd)");
+ const isWindow = f(u, "int __stdcall IsWindow(uintptr_t hwnd)");
+ const iconic = f(u, "int __stdcall IsIconic(uintptr_t hwnd)");
+ const pidOf = f(u, "uint32_t __stdcall GetWindowThreadProcessId(uintptr_t hwnd, _Out_ uint32_t *pid)");
+ const titleOf = f(u, "int __stdcall GetWindowTextW(uintptr_t hwnd, _Out_ uint16_t *text, int count)");
+ const boundsOf = f(dwm, "int32_t __stdcall DwmGetWindowAttribute(uintptr_t hwnd, uint32_t attribute, _Out_ CU_RECT *rect, uint32_t size)");
+ const foreground = f(u, "uintptr_t __stdcall GetForegroundWindow()");
+ const ancestor = f(u, "uintptr_t __stdcall GetAncestor(uintptr_t hwnd, uint32_t flag)");
+ const fromPoint = f(u, "uintptr_t __stdcall WindowFromPoint(CU_POINT point)");
+ const focus = f(u, "int __stdcall SetForegroundWindow(uintptr_t hwnd)");
+ const send = f(u, "uint32_t __stdcall SendInput(uint32_t count, CU_INPUT *inputs, int size)");
+ const metrics = f(u, "int __stdcall GetSystemMetrics(int index)");
+ const openProcess = f(k, "uintptr_t __stdcall OpenProcess(uint32_t access, int inherit, uint32_t pid)");
+ const processName = f(k, "int __stdcall QueryFullProcessImageNameW(uintptr_t process, uint32_t flags, _Out_ uint16_t *name, _Inout_ uint32_t *size)");
+ const times = f(k, "int __stdcall GetProcessTimes(uintptr_t process, _Out_ CU_FILETIME *created, _Out_ CU_FILETIME *exited, _Out_ CU_FILETIME *kernel, _Out_ CU_FILETIME *user)");
+ const closeHandle = f(k, "int __stdcall CloseHandle(uintptr_t handle)");
+ const openDesktop = f(u, "uintptr_t __stdcall OpenInputDesktop(uint32_t flags, int inherit, uint32_t access)");
+ const closeDesktop = f(u, "int __stdcall CloseDesktop(uintptr_t desktop)");
+ const userInfo = f(u, "int __stdcall GetUserObjectInformationW(uintptr_t object, int index, _Out_ uint16_t *value, uint32_t size, _Out_ uint32_t *needed)");
+ f(u, "intptr_t __stdcall SetThreadDpiAwarenessContext(intptr_t context)")(-4);
+ const marker = 0x4f4d4e49;
+ const inject = (value) => { if (send(1, [value], koffi.sizeof(input)) !== 1) fail("input_unavailable", "Windows blocked input. Keep the app on the active desktop, run it without administrator privileges, and continue."); };
+ const keyEvent = (vk, scan, flags) => inject({ type: 1, value: { ki: { wVk: vk, wScan: scan, dwFlags: flags, time: 0, dwExtraInfo: marker } } });
+ const mouseEvent = (flags, data = 0, dx = 0, dy = 0) => inject({ type: 0, value: { mi: { dx, dy, mouseData: data >>> 0, dwFlags: flags, time: 0, dwExtraInfo: marker } } });
+ const downKeys = new Set();
+ let pointerDown = false;
+ const release = () => {
+ for (const key of downKeys) { try { keyEvent(key, 0, 2); } catch {} }
+ downKeys.clear();
+ if (pointerDown) { try { mouseEvent(4); } catch {} pointerDown = false; }
+ };
+ const peek = f(u, "int __stdcall PeekMessageW(_Out_ CU_MSG *message, uintptr_t hwnd, uint32_t first, uint32_t last, uint32_t remove)");
+ const nextHook = f(u, "intptr_t __stdcall CallNextHookEx(uintptr_t hook, int code, uintptr_t wParam, intptr_t lParam)");
+ const installHook = f(u, "uintptr_t __stdcall SetWindowsHookExW(int type, CU_HOOKPROC *callback, uintptr_t module, uint32_t thread)");
+ const uninstall = f(u, "int __stdcall UnhookWindowsHookEx(uintptr_t hook)");
+ const callbacks = [hookKey, hookMouse].map((type) => koffi.register((code, wParam, lParam) => {
+ if (code >= 0) {
+ const value = koffi.decode(lParam, type);
+ if (Number(value.dwExtraInfo) !== marker) { changed(); release(); }
+ }
+ return nextHook(0, code, wParam, lParam);
+ }, koffi.pointer(hookProto)));
+ const hooks = [installHook(13, callbacks[0], 0, 0), installHook(14, callbacks[1], 0, 0)];
+ if (hooks.some((v) => !v)) { hooks.forEach((h) => { if (h) uninstall(h); }); callbacks.forEach((c) => koffi.unregister(c)); fail("input_unavailable", "Windows input monitoring is unavailable. Restart OmniRush.ai on your normal desktop."); }
+ const poll = () => { const value = {}; while (peek(value, 0, 0, 0, 1)) {} };
+ const available = () => {
+ const handle = openDesktop(0, 0, 1);
+ if (!handle) return false;
+ try { const data = Buffer.alloc(512), size = [0]; return Boolean(userInfo(handle, 2, data, data.length, size)) && data.toString("utf16le").split("\0")[0].toLowerCase() === "default"; }
+ finally { closeDesktop(handle); }
+ };
+ function window(id) {
+ if (!available() || !isWindow(id) || !visible(id) || iconic(id)) fail("window_unavailable", "Keep the approved app visible on an unlocked desktop.");
+ const pid = [0]; pidOf(id, pid);
+ const handle = openProcess(0x1000, 0, pid[0]);
+ if (!handle) fail("window_unavailable", "This app cannot be inspected. Run it without administrator privileges.");
+ let executable, identity;
+ try {
+ const name = Buffer.alloc(65536), size = [32768], created = {};
+ if (!processName(handle, 0, name, size) || !times(handle, created, {}, {}, {})) fail("window_unavailable", "The app identity could not be verified.");
+ executable = name.toString("utf16le", 0, size[0] * 2);
+ identity = String(pid[0]) + ":" + created.high + ":" + created.low;
+ } finally { closeHandle(handle); }
+ const box = {}, title = Buffer.alloc(8192);
+ if (boundsOf(id, 9, box, koffi.sizeof(rect)) !== 0) fail("window_unavailable", "The window bounds could not be read.");
+ titleOf(id, title, 4096);
+ const bounds = { x: box.left, y: box.top, width: box.right - box.left, height: box.bottom - box.top };
+ if (bounds.width <= 0 || bounds.height <= 0) fail("window_unavailable", "The window is not visible.");
+ return safeWindow({ id: Number(id), pid: pid[0], identity, executable, appId: executable.toLowerCase(), appName: path.win32.basename(executable, ".exe"), title: title.toString("utf16le").split("\0")[0], bounds });
+ }
+ function guard(expected, expectedGeneration, pointValue, deadline) {
+ if (Date.now() >= deadline) fail("session_expired", "The approved session expired. Start a new session.");
+ poll();
+ if (generation !== expectedGeneration) fail("user_interacting", "Your input interrupted Computer Use. Choose Continue when ready.");
+ const current = window(expected.id);
+ if (current.identity !== expected.identity || JSON.stringify(current.bounds) !== JSON.stringify(expected.bounds)) fail("window_changed", "The approved window changed. Observe it again.");
+ if (Number(ancestor(foreground(), 2)) !== expected.id) fail("focus_changed", "The approved window must stay in front. Choose Continue when ready.");
+ if (pointValue && Number(ancestor(fromPoint(pointValue), 2)) !== expected.id) fail("window_occluded", "Another window covers this point. Move it away and continue.");
+ return current;
+ }
+ const move = (x, y) => {
+ const left = metrics(76), top = metrics(77), width = metrics(78), height = metrics(79);
+ if (width <= 1 || height <= 1) fail("input_unavailable", "The desktop bounds could not be read.");
+ mouseEvent(0x8000 | 0x4000 | 1, 0, Math.round((x - left) * 65535 / (width - 1)), Math.round((y - top) * 65535 / (height - 1)));
+ };
+ return {
+ poll, window, available, release, check: guard,
+ list() { const result = []; enumWindows((id) => { try { const item = window(id); if (item.title) result.push(item); } catch {} return 1; }, 0); return result; },
+ focus(expected) { if (window(expected.id).identity !== expected.identity || !focus(expected.id)) fail("focus_changed", "Bring the approved app to the front, then choose Continue."); poll(); },
+ async act(expected, expectedGeneration, action, deadline) {
+ const checked = (p) => guard(expected, expectedGeneration, p, deadline);
+ const pointValue = (p) => ({ x: expected.bounds.x + Math.round(p.x), y: expected.bounds.y + Math.round(p.y) });
+ try {
+ checked();
+ if (action.type === "type") {
+ for (const char of action.text) {
+ checked();
+ for (let i = 0; i < char.length; i++) { keyEvent(0, char.charCodeAt(i), 4); keyEvent(0, char.charCodeAt(i), 6); }
+ await new Promise((resolve) => setTimeout(resolve, 2));
+ }
+ } else if (action.type === "key") {
+ const modifiers = keys(action), vk = keyNames[action.key] ?? (/^F/.test(action.key) ? 111 + Number(action.key.slice(1)) : action.key.toUpperCase().charCodeAt(0));
+ for (const modifier of modifiers) { const key = { ctrl: 17, shift: 16, alt: 18 }[modifier]; downKeys.add(key); keyEvent(key, 0, 0); }
+ checked(); downKeys.add(vk); keyEvent(vk, 0, 0); keyEvent(vk, 0, 2); downKeys.delete(vk);
+ } else if (["click", "double_click", "scroll", "drag"].includes(action.type)) {
+ const points = action.type === "drag" ? action.path : [action];
+ for (const p of points) {
+ const position = pointValue(p); checked(position); move(position.x, position.y); checked(position);
+ if (action.type === "drag" && !pointerDown) { pointerDown = true; mouseEvent(2); }
+ await new Promise((resolve) => setTimeout(resolve, action.type === "drag" ? 12 : 2));
+ }
+ const position = pointValue(points.at(-1)); checked(position);
+ if (action.type === "scroll") mouseEvent(action.axis === "horizontal" ? 0x1000 : 0x800, action.delta * (action.axis === "horizontal" ? 120 : -120));
+ else if (action.type !== "drag") {
+ for (let i = 0; i < (action.type === "double_click" ? 2 : 1); i++) {
+ checked(position); pointerDown = true; mouseEvent(2); mouseEvent(4); pointerDown = false;
+ if (i === 0 && action.type === "double_click") await new Promise((resolve) => setTimeout(resolve, 60));
+ }
+ }
+ } else fail("unsupported_action", "Use click, double_click, drag, scroll, type, or key.");
+ } finally { release(); }
+ },
+ close() { release(); hooks.forEach((hook) => uninstall(hook)); callbacks.forEach((c) => koffi.unregister(c)); },
+ };
+}
+
+function x11() {
+ if (process.env.XDG_SESSION_TYPE === "wayland" || process.env.WAYLAND_DISPLAY) fail("wayland_unavailable", "Wayland needs a desktop portal connection. This build supports Linux X11; sign in to an X11 session or use the built-in browser and app connectors.");
+ if (!process.env.DISPLAY) fail("desktop_unavailable", "Computer Use needs a graphical Linux desktop.");
+ const x = koffi.load("libX11.so.6"), xt = koffi.load("libXtst.so.6"), xi = koffi.load("libXi.so.6");
+ const attributes = koffi.struct("CU_XWindowAttributes", {
+ x: "int", y: "int", width: "int", height: "int", border_width: "int", depth: "int", visual: "void *", root: "ulong", class: "int", bit_gravity: "int", win_gravity: "int", backing_store: "int", backing_planes: "ulong", backing_pixel: "ulong", save_under: "int", colormap: "ulong", map_installed: "int", map_state: "int", all_event_masks: "long", your_event_mask: "long", do_not_propagate_mask: "long", override_redirect: "int", screen: "void *",
+ });
+ const cookieType = koffi.struct("CU_XGenericEventCookie", { type: "int", serial: "ulong", send_event: "int", display: "void *", extension: "int", evtype: "int", cookie: "uint", data: "void *" });
+ const rawType = koffi.struct("CU_XIRawEvent", { type: "int", serial: "ulong", send_event: "int", display: "void *", extension: "int", evtype: "int", time: "ulong", deviceid: "int", sourceid: "int", detail: "int", flags: "int", valuators: koffi.struct({ mask_len: "int", mask: "void *", values: "void *" }), raw_values: "void *" });
+ const deviceType = koffi.struct("CU_XIDeviceInfo", { deviceid: "int", name: "str", use: "int", attachment: "int", enabled: "int", num_classes: "int", classes: "void *" });
+ const maskType = koffi.struct("CU_XIEventMask", { deviceid: "int", mask_len: "int", mask: "void *" });
+ const open = x.func("void *XOpenDisplay(str name)"), close = x.func("int XCloseDisplay(void *display)");
+ const display = open(null);
+ if (!display) fail("desktop_unavailable", "Could not connect to the X11 desktop.");
+ const root = x.func("ulong XDefaultRootWindow(void *display)")(display);
+ const atom = (name) => x.func("ulong XInternAtom(void *display, str name, int only)")(display, name, 0);
+ const property = x.func("int XGetWindowProperty(void *display, ulong window, ulong property, long offset, long length, int remove, ulong requested, _Out_ ulong *actual, _Out_ int *format, _Out_ ulong *count, _Out_ ulong *remaining, _Out_ void **data)");
+ const free = x.func("int XFree(void *data)");
+ const boundsOf = x.func("int XGetWindowAttributes(void *display, ulong window, _Out_ CU_XWindowAttributes *attributes)");
+ const translate = x.func("int XTranslateCoordinates(void *display, ulong source, ulong destination, int x, int y, _Out_ int *root_x, _Out_ int *root_y, _Out_ ulong *child)");
+ const queryTree = x.func("int XQueryTree(void *display, ulong window, _Out_ ulong *root, _Out_ ulong *parent, _Out_ void **children, _Out_ uint *count)");
+ const sync = x.func("int XSync(void *display, int discard)");
+ const setError = x.func("void *XSetErrorHandler(void *callback)");
+ const errorProto = koffi.proto("int CU_XErrorHandler(void *display, void *event)");
+ let xError = false;
+ const errorCallback = koffi.register(() => { xError = true; return 0; }, koffi.pointer(errorProto));
+ setError(errorCallback);
+ const pending = x.func("int XPending(void *display)"), nextEvent = x.func("int XNextEvent(void *display, void *event)");
+ const eventData = x.func("int XGetEventData(void *display, _Inout_ CU_XGenericEventCookie *cookie)");
+ const freeEventData = x.func("void XFreeEventData(void *display, CU_XGenericEventCookie *cookie)");
+ const queryDevices = xi.func("CU_XIDeviceInfo *XIQueryDevice(void *display, int deviceid, _Out_ int *count)");
+ const freeDevices = xi.func("void XIFreeDeviceInfo(CU_XIDeviceInfo *devices)");
+ const selectEvents = xi.func("int XISelectEvents(void *display, ulong window, CU_XIEventMask *masks, int count)");
+ const major = [2], minor = [0];
+ if (xi.func("int XIQueryVersion(void *display, _Inout_ int *major, _Inout_ int *minor)")(display, major, minor) !== 0) fail("input_unavailable", "XInput2 is required for interruption monitoring.");
+ const testDevices = new Set();
+ const refreshDevices = () => { const count = [0], ptr = queryDevices(display, 0, count); testDevices.clear(); if (ptr) { for (const d of koffi.decode(ptr, deviceType, count[0])) if (d.name.includes("XTEST")) testDevices.add(d.deviceid); freeDevices(ptr); } };
+ refreshDevices();
+ const eventMask = Buffer.alloc(4);
+ for (const type of [11, 13, 14, 15, 16, 17]) eventMask[type >> 3] |= 1 << (type & 7);
+ if (selectEvents(display, root, [{ deviceid: 0, mask_len: eventMask.length, mask: eventMask }], 1) !== 0) fail("input_unavailable", "X11 input monitoring could not start.");
+ sync(display, 0);
+ let injecting = false;
+ const poll = () => {
+ while (pending(display)) {
+ const event = Buffer.alloc(192); nextEvent(display, event);
+ const cookie = koffi.decode(event, cookieType);
+ if (cookie.type !== 35 || !eventData(display, cookie)) continue;
+ try {
+ if (cookie.evtype === 11) refreshDevices();
+ else if ([13, 14, 15, 16, 17].includes(cookie.evtype)) {
+ const raw = koffi.decode(cookie.data, rawType);
+ if (!injecting || !testDevices.has(raw.sourceid)) changed();
+ }
+ } finally { freeEventData(display, cookie); }
+ }
+ };
+ function prop(id, name) {
+ const actual = [0], format = [0], count = [0], remaining = [0], data = [null];
+ if (property(display, id, atom(name), 0, 4096, 0, 0, actual, format, count, remaining, data) !== 0 || !data[0]) return [];
+ try {
+ if (format[0] === 32) return Array.from(koffi.decode(data[0], "ulong", Math.min(count[0], 4096)), Number);
+ if (format[0] === 8) return Buffer.from(koffi.decode(data[0], "uint8_t", Math.min(count[0], 16384))).toString("utf8");
+ return [];
+ } finally { free(data[0]); }
+ }
+ const tree = (id) => {
+ const parent = [0], parentRoot = [0], data = [null], count = [0];
+ if (!queryTree(display, id, parentRoot, parent, data, count)) return { parent: 0, children: [] };
+ try { return { parent: Number(parent[0]), children: data[0] ? Array.from(koffi.decode(data[0], "ulong", Math.min(count[0], 4096)), Number) : [] }; }
+ finally { if (data[0]) free(data[0]); }
+ };
+ const owner = (id, target) => { for (let i = 0; id && i < 32; i++) { if (Number(id) === target) return true; id = tree(id).parent; } return false; };
+ function window(id) {
+ const box = {};
+ if (!boundsOf(display, id, box) || box.map_state !== 2 || box.width <= 0 || box.height <= 0) fail("window_unavailable", "Keep the approved app visible on your desktop.");
+ const pid = prop(id, "_NET_WM_PID")[0];
+ let executable, identity;
+ try {
+ executable = readlinkSync("/proc/" + pid + "/exe");
+ const stat = readFileSync("/proc/" + pid + "/stat", "utf8");
+ identity = String(pid) + ":" + stat.slice(stat.lastIndexOf(")") + 2).split(" ")[19];
+ } catch { fail("window_unavailable", "The app identity could not be verified."); }
+ const xRoot = [0], yRoot = [0], child = [0];
+ if (!translate(display, id, root, 0, 0, xRoot, yRoot, child)) fail("window_unavailable", "Could not read the window position.");
+ const title = prop(id, "_NET_WM_NAME") || prop(id, "WM_NAME");
+ return safeWindow({ id: Number(id), pid: Number(pid), executable, identity, appId: executable, appName: path.basename(executable), title: typeof title === "string" ? title.replaceAll("\0", "") : "App window", bounds: { x: xRoot[0], y: yRoot[0], width: box.width, height: box.height } });
+ }
+ const currentFocus = () => { const focused = [0], revert = [0]; x.func("int XGetInputFocus(void *display, _Out_ ulong *focus, _Out_ int *revert)")(display, focused, revert); return Number(focused[0]); };
+ function guard(expected, expectedGeneration, position, deadline) {
+ if (Date.now() >= deadline) fail("session_expired", "The approved session expired. Start a new session.");
+ poll();
+ if (generation !== expectedGeneration) fail("user_interacting", "Your input interrupted Computer Use. Choose Continue when ready.");
+ const current = window(expected.id);
+ if (current.identity !== expected.identity || JSON.stringify(current.bounds) !== JSON.stringify(expected.bounds)) fail("window_changed", "The approved window changed. Observe it again.");
+ if (!owner(currentFocus(), expected.id)) fail("focus_changed", "The approved window must stay in front. Choose Continue when ready.");
+ if (position) {
+ const children = [], outputRoot = [0], child = [0], rx = [0], ry = [0], wx = [0], wy = [0], mask = [0];
+ let parent = root;
+ const queryPointer = x.func("int XQueryPointer(void *display, ulong window, _Out_ ulong *root, _Out_ ulong *child, _Out_ int *root_x, _Out_ int *root_y, _Out_ int *win_x, _Out_ int *win_y, _Out_ uint *mask)");
+ for (let i = 0; i < 32; i++) { if (!queryPointer(display, parent, outputRoot, child, rx, ry, wx, wy, mask) || !child[0]) break; parent = Number(child[0]); children.push(parent); }
+ if (!children.includes(expected.id)) fail("window_occluded", "Another window covers the pointer. Move it away and continue.");
+ }
+ return current;
+ }
+ // Capture the approved window's off-screen pixmap. XGetImage on the visible
+ // window can include another app covering it on a non-compositing desktop.
+ const composite = koffi.load("libXcomposite.so.1");
+ const compositeMajor = [0], compositeMinor = [0];
+ if (!composite.func("int XCompositeQueryVersion(void *display, _Out_ int *major, _Out_ int *minor)")(display, compositeMajor, compositeMinor) || (compositeMajor[0] === 0 && compositeMinor[0] < 2)) fail("capture_unavailable", "X11 Composite window capture is required.");
+ const redirect = composite.func("void XCompositeRedirectWindow(void *display, ulong window, int update)");
+ const unredirect = composite.func("void XCompositeUnredirectWindow(void *display, ulong window, int update)");
+ const namePixmap = composite.func("ulong XCompositeNameWindowPixmap(void *display, ulong window)");
+ const imageType = koffi.struct("CU_XImage", { width: "int", height: "int", xoffset: "int", format: "int", data: "void *", byte_order: "int", bitmap_unit: "int", bitmap_bit_order: "int", bitmap_pad: "int", depth: "int", bytes_per_line: "int", bits_per_pixel: "int", red_mask: "ulong", green_mask: "ulong", blue_mask: "ulong", obdata: "void *", functions: koffi.struct({ create_image: "void *", destroy_image: "void *", get_pixel: "void *", put_pixel: "void *", sub_image: "void *", add_pixel: "void *" }) });
+ const visualType = koffi.struct("CU_XVisual", { ext_data: "void *", visualid: "ulong", class: "int", red_mask: "ulong", green_mask: "ulong", blue_mask: "ulong", bits_per_rgb: "int", map_entries: "int" });
+ const getImage = x.func("CU_XImage *XGetImage(void *display, ulong drawable, int x, int y, uint width, uint height, ulong planes, int format)");
+ const destroyImage = x.func("int XDestroyImage(CU_XImage *image)");
+ const freePixmap = x.func("int XFreePixmap(void *display, ulong pixmap)");
+ const redirected = new Set();
+ const clearCapture = () => { for (const id of redirected) unredirect(display, id, 0); redirected.clear(); sync(display, 0); };
+ const pngChunk = (type, data) => { const name = Buffer.from(type), length = Buffer.alloc(4), checksum = Buffer.alloc(4); length.writeUInt32BE(data.length); checksum.writeUInt32BE(crc32(data, crc32(name))); return Buffer.concat([length, name, data, checksum]); };
+ async function capture(expected) {
+ const current = window(expected.id);
+ if (current.identity !== expected.identity || JSON.stringify(current.bounds) !== JSON.stringify(expected.bounds)) fail("window_changed", "The window changed. Observe again.");
+ if (current.bounds.width * current.bounds.height > 16_777_216) fail("capture_unavailable", "This window is too large to capture. Resize it and observe again.");
+ if (!redirected.has(current.id)) {
+ xError = false; redirect(display, current.id, 0); sync(display, 0);
+ if (xError) fail("capture_unavailable", "The approved window cannot be captured separately.");
+ redirected.add(current.id);
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ }
+ xError = false;
+ const pixmap = namePixmap(display, current.id); sync(display, 0);
+ if (xError || !pixmap) fail("capture_unavailable", "The approved window cannot be captured separately.");
+ let ptr;
+ try {
+ ptr = getImage(display, pixmap, 0, 0, current.bounds.width, current.bounds.height, ~0n, 2);
+ if (!ptr) fail("capture_unavailable", "The window image is unavailable. Keep it visible and observe again.");
+ const value = koffi.decode(ptr, imageType), bytes = value.bits_per_pixel / 8;
+ const attributesValue = {};
+ if (!boundsOf(display, current.id, attributesValue) || !attributesValue.visual) fail("capture_unavailable", "The window color format is unavailable.");
+ const visual = koffi.decode(attributesValue.visual, visualType);
+ if (![2, 3, 4].includes(bytes) || value.width !== current.bounds.width || value.height !== current.bounds.height || value.bytes_per_line < value.width * bytes || value.bytes_per_line * value.height > 67_108_864 || ![0, 1].includes(value.byte_order) || visual.class !== 4 || value.depth !== attributesValue.depth || !visual.red_mask || !visual.green_mask || !visual.blue_mask) fail("capture_unavailable", "This X11 image format is unavailable.");
+ const source = Buffer.from(koffi.decode(value.data, "uint8_t", value.bytes_per_line * value.height));
+ const ratio = Math.min(1, 1600 / Math.max(value.width, value.height));
+ const width = Math.max(1, Math.floor(value.width * ratio)), height = Math.max(1, Math.floor(value.height * ratio));
+ const rows = Buffer.alloc((width * 4 + 1) * height);
+ const channels = [visual.red_mask, visual.green_mask, visual.blue_mask].map((mask) => ({ mask, shift: Math.log2((mask & -mask) >>> 0), max: mask >>> Math.log2((mask & -mask) >>> 0) }));
+ for (let y = 0; y < height; y++) for (let column = 0; column < width; column++) {
+ const offset = Math.floor(y / ratio) * value.bytes_per_line + Math.floor(column / ratio) * bytes;
+ const pixel = value.byte_order === 0 ? source.readUIntLE(offset, bytes) : source.readUIntBE(offset, bytes);
+ const destination = y * (width * 4 + 1) + 1 + column * 4;
+ channels.forEach((channel, index) => { rows[destination + index] = Math.round(((pixel & channel.mask) >>> channel.shift) * 255 / channel.max); });
+ rows[destination + 3] = 255;
+ }
+ const header = Buffer.alloc(13); header.writeUInt32BE(width); header.writeUInt32BE(height, 4); header[8] = 8; header[9] = 6;
+ return { image: Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), pngChunk("IHDR", header), pngChunk("IDAT", deflateSync(rows, { level: 1 })), pngChunk("IEND", Buffer.alloc(0))]), size: { width, height } };
+ } finally { if (ptr) destroyImage(ptr); freePixmap(display, pixmap); }
+ }
+ const rawKey = xt.func("int XTestFakeKeyEvent(void *display, uint key, int down, ulong delay)");
+ const rawButton = xt.func("int XTestFakeButtonEvent(void *display, uint button, int down, ulong delay)");
+ const rawMotion = xt.func("int XTestFakeMotionEvent(void *display, int screen, int x, int y, ulong delay)");
+ const inject = (fn, args, check = true) => {
+ const previous = generation; poll();
+ if (check && previous !== generation) fail("user_interacting", "Your input interrupted Computer Use. Choose Continue when ready.");
+ injecting = true;
+ try { const result = fn(...args); sync(display, 0); poll(); return result; }
+ finally { injecting = false; }
+ };
+ const fakeKey = (...args) => inject(rawKey, args);
+ const fakeButton = (...args) => inject(rawButton, args);
+ const fakeMotion = (...args) => inject(rawMotion, args);
+ const keysym = x.func("ulong XStringToKeysym(str name)"), keycode = x.func("uint8_t XKeysymToKeycode(void *display, ulong symbol)");
+ const downKeys = new Set();
+ let pointerDown = false;
+ const release = () => { for (const key of downKeys) inject(rawKey, [display, key, 0, 0], false); downKeys.clear(); if (pointerDown) inject(rawButton, [display, 1, 0, 0], false); pointerDown = false; sync(display, 0); };
+ const symbolAt = x.func("ulong XKeycodeToKeysym(void *display, uint8_t code, int index)");
+ const key = (name, down) => { const symbol = typeof name === "number" ? name : keysym(name); const code = keycode(display, symbol); if (!code) fail("unsupported_action", "This character is unavailable in the active keyboard layout. Use an app connector for arbitrary Unicode text."); if (down) downKeys.add(code); else downKeys.delete(code); if (!fakeKey(display, code, down ? 1 : 0, 0)) fail("input_unavailable", "X11 rejected keyboard input."); sync(display, 0); };
+ return {
+ poll, window, release, check: guard, capture, clearCapture, available: () => true,
+ list() { const ids = prop(root, "_NET_CLIENT_LIST_STACKING"); const result = []; for (const id of Array.isArray(ids) && ids.length ? ids : tree(root).children) { try { result.push(window(id)); } catch {} } return result; },
+ focus(expected) {
+ if (window(expected.id).identity !== expected.identity) fail("window_unavailable", "The approved app has closed.");
+ x.func("int XRaiseWindow(void *display, ulong window)")(display, expected.id);
+ x.func("int XSetInputFocus(void *display, ulong window, int revert, ulong time)")(display, expected.id, 2, 0);
+ sync(display, 0); poll();
+ },
+ async act(expected, expectedGeneration, action, deadline) {
+ const checked = (position) => guard(expected, expectedGeneration, position, deadline);
+ try {
+ checked();
+ if (action.type === "type") {
+ for (const char of action.text) {
+ checked();
+ const cp = char.codePointAt(0);
+ const symbol = char === "\n" ? keysym("Return") : char === "\t" ? keysym("Tab") : cp <= 255 ? cp : 0x01000000 + cp;
+ const code = keycode(display, symbol);
+ if (!code || (Number(symbolAt(display, code, 0)) !== symbol && Number(symbolAt(display, code, 1)) !== symbol)) fail("unsupported_action", "This character is unavailable in the active keyboard layout. Use an app connector for arbitrary Unicode text.");
+ const shifted = Number(symbolAt(display, code, 1)) === symbol && Number(symbolAt(display, code, 0)) !== symbol;
+ if (shifted) key("Shift_L", true);
+ key(symbol, true); key(symbol, false);
+ if (shifted) key("Shift_L", false);
+ await new Promise((resolve) => setTimeout(resolve, 2));
+ }
+ } else if (action.type === "key") {
+ const modifiers = keys(action), name = { Enter: "Return", Space: "space", PageUp: "Prior", PageDown: "Next" }[action.key] ?? action.key;
+ for (const modifier of modifiers) key({ ctrl: "Control_L", shift: "Shift_L", alt: "Alt_L" }[modifier], true);
+ checked(); key(name, true); key(name, false);
+ } else if (["click", "double_click", "scroll", "drag"].includes(action.type)) {
+ const points = action.type === "drag" ? action.path : [action];
+ for (const p of points) {
+ checked(); fakeMotion(display, -1, expected.bounds.x + Math.round(p.x), expected.bounds.y + Math.round(p.y), 0); sync(display, 0); checked(p);
+ if (action.type === "drag" && !pointerDown) { pointerDown = true; fakeButton(display, 1, 1, 0); sync(display, 0); }
+ await new Promise((resolve) => setTimeout(resolve, action.type === "drag" ? 12 : 2));
+ }
+ checked(points.at(-1));
+ if (action.type === "scroll") {
+ const button = action.axis === "horizontal" ? action.delta > 0 ? 7 : 6 : action.delta > 0 ? 5 : 4;
+ for (let i = 0; i < Math.abs(action.delta); i++) { checked(points.at(-1)); fakeButton(display, button, 1, 0); fakeButton(display, button, 0, 0); sync(display, 0); }
+ } else if (action.type !== "drag") {
+ for (let i = 0; i < (action.type === "double_click" ? 2 : 1); i++) { checked(points.at(-1)); pointerDown = true; fakeButton(display, 1, 1, 0); fakeButton(display, 1, 0, 0); pointerDown = false; sync(display, 0); if (i === 0 && action.type === "double_click") await new Promise((resolve) => setTimeout(resolve, 60)); }
+ }
+ } else fail("unsupported_action", "Use click, double_click, drag, scroll, type, or key.");
+ } finally { release(); }
+ },
+ close() { release(); clearCapture(); close(display); setError(null); koffi.unregister(errorCallback); },
+ };
+}
+let timer;
+try {
+ backend = process.platform === "win32" ? windows() : x11();
+ timer = setInterval(() => { try { backend.poll(); } catch { changed(); } }, 10);
+ parentPort.postMessage({ event: "ready", supported: true });
+} catch (error) {
+ parentPort.postMessage({ event: "ready", supported: false, code: error.code ?? "native_unavailable", error: error.message });
+}
+let busy = false;
+parentPort.on("message", async ({ id, method, params }) => {
+ let ownsBusy = false;
+ try {
+ if (method === "shutdown") {
+ generation++; backend?.release(); clearInterval(timer);
+ while (busy) await new Promise((resolve) => setTimeout(resolve, 10));
+ backend?.close(); backend = null;
+ parentPort.postMessage({ id, result: {} }); parentPort.close(); return;
+ }
+ if (!backend) fail("native_unavailable", "Desktop access is unavailable.");
+ if (method === "stop") { generation++; backend.release(); backend.clearCapture?.(); parentPort.postMessage({ id, result: {} }); return; }
+ if (busy) fail("busy", "Desktop operations must be sequential.");
+ busy = true; ownsBusy = true;
+ backend.poll();
+ let result;
+ if (method === "list") result = backend.list();
+ else if (method === "window") result = backend.window(params.id);
+ else if (method === "focus") { backend.focus(params.window); result = { generation }; }
+ else if (method === "state") result = { generation, available: backend.available() };
+ else if (method === "capture") { result = await backend.capture(params.window); }
+ else if (method === "check") { backend.check(params.window, params.generation, undefined, params.deadline); result = {}; }
+ else if (method === "act") { await backend.act(params.window, params.generation, params.action, params.deadline); result = { generation }; }
+ else fail("unknown_method", "Unknown native operation.");
+ parentPort.postMessage({ id, result });
+ } catch (error) {
+ parentPort.postMessage({ id, error: { code: error.code ?? "native_error", message: error.message } });
+ } finally { if (ownsBusy) busy = false; }
+});
+parentPort.on("close", () => { clearInterval(timer); backend?.close(); });
diff --git a/apps/desktop/electron/computer-use-native.mjs b/apps/desktop/electron/computer-use-native.mjs
new file mode 100644
index 00000000..de518abf
--- /dev/null
+++ b/apps/desktop/electron/computer-use-native.mjs
@@ -0,0 +1,45 @@
+import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+/** @param {unknown} value @returns {value is Record<string, unknown>} */
+function record(value) { return typeof value === "object" && value !== null && !Array.isArray(value); }
+export function createNativeDesktop() {
+ const worker = spawn(process.execPath, [fileURLToPath(new URL("./computer-use-native-worker.mjs", import.meta.url))], { env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" }, stdio: ["ignore", "ignore", "ignore", "ipc"], windowsHide: true, serialization: "advanced" });
+ const pending = new Map();
+ const listeners = new Set();
+ let nextId = 0, closed = false;
+ let readyResolve;
+ const ready = new Promise((resolve) => { readyResolve = resolve; });
+ const readyTimer = setTimeout(() => readyResolve({ supported: false, error: "Desktop access did not start. Restart OmniRush.ai." }), 10_000);
+ worker.on("message", (value) => {
+ if (!record(value)) return;
+ if (value.event === "ready") { clearTimeout(readyTimer); readyResolve(value); }
+ if (value.event === "person-input") for (const listener of listeners) listener(value);
+ const request = pending.get(value.id);
+ if (!request) return;
+ pending.delete(value.id); clearTimeout(request.timer);
+ if (record(value.error) && typeof value.error.message === "string") request.reject(Object.assign(new Error(value.error.message), { code: value.error.code }));
+ else request.resolve(value.result);
+ });
+ const fail = (error) => {
+ clearTimeout(readyTimer); readyResolve({ supported: false, error: error.message });
+ closed = true;
+ for (const request of pending.values()) { clearTimeout(request.timer); request.reject(error); }
+ pending.clear();
+ for (const listener of listeners) listener({ event: "unavailable" });
+ };
+ worker.on("error", fail);
+ worker.on("exit", () => fail(new Error("Desktop access stopped. Restart OmniRush.ai.")));
+ return {
+ ready,
+ onInput(listener) { listeners.add(listener); return () => listeners.delete(listener); },
+ call(method, params = {}) {
+ if (closed) return Promise.reject(new Error("Desktop access has stopped."));
+ const id = ++nextId;
+ return new Promise((resolve, reject) => {
+ const timer = setTimeout(() => { pending.delete(id); reject(new Error("Desktop operation timed out.")); }, 10_000);
+ pending.set(id, { resolve, reject, timer }); worker.send({ id, method, params });
+ });
+ },
+ async close() { if (closed) return; await this.call("shutdown").catch(() => {}); closed = true; if (worker.exitCode === null) worker.kill(); },
+ };
+}
diff --git a/apps/desktop/electron/computer-use-portable-host.mjs b/apps/desktop/electron/computer-use-portable-host.mjs
new file mode 100644
index 00000000..3469a07e
--- /dev/null
+++ b/apps/desktop/electron/computer-use-portable-host.mjs
@@ -0,0 +1,246 @@
+// Windows / X11 use the same MCP contract and main-window-only approvals as Mac.
+// The private native bridge is never reachable over the MCP transport.
+import { createServer } from "node:net";
+import { randomBytes, randomUUID, createHash, timingSafeEqual } from "node:crypto";
+import { mkdir, writeFile, rename, unlink } from "node:fs/promises";
+import path from "node:path";
+import { createNativeDesktop } from "./computer-use-native.mjs";
+
+const modes = ["observe", "control"];
+const methods = new Set(["initialize", "ping", "tools/list", "tools/call", "notifications/initialized", "notifications/cancelled"]);
+const tool = (name, description, properties, required = []) => ({ name, description, inputSchema: { type: "object", properties, required, additionalProperties: false } });
+const string = { type: "string" };
+const session = { session_id: string };
+const tools = [
+ tool("computer_discover", "List running app identities without window contents. Start an app yourself if it is closed. Windows and Linux X11 support observe and foreground control; assist is unavailable.", {}),
+ tool("computer_open_session", "Ask the person to approve one visible app window. Never infer consent. Wait for this tool to return before observing.", { app_id: string, pid: { type: "integer" }, mode: { type: "string", enum: modes }, purpose: string }, ["app_id", "purpose"]),
+ tool("computer_observe", "Get a fresh screenshot of the approved window. Visual models are required. Coordinates for actions are screenshot pixels, relative to the window. Observe again after every action or failure.", { ...session, include_image: { type: "boolean" } }, ["session_id"]),
+ tool("computer_act", "Perform one action using a fresh, single-use observation. Supported: click/double_click (x,y), drag (path [{x,y}]), scroll (x,y,delta in steps (positive down/right),axis), type (text), key (key,modifiers ctrl/shift/alt). Foreground only. Never retry an action when may_have_acted is true; observe and inspect its result.", { ...session, observation_id: string, request_id: string, action: { type: "object", properties: { type: { type: "string", enum: ["click", "double_click", "drag", "scroll", "type", "key"] }, x: { type: "number" }, y: { type: "number" }, path: { type: "array", items: { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false } }, delta: { type: "integer" }, axis: { type: "string", enum: ["vertical", "horizontal"] }, text: string, key: string, modifiers: { type: "array", items: { type: "string", enum: ["ctrl", "shift", "alt"] } } }, required: ["type"], additionalProperties: false } }, ["session_id", "observation_id", "request_id", "action"]),
+ tool("computer_session_status", "Read the session status. After person input, wait for the person to choose Continue; the agent cannot resume itself.", session, ["session_id"]),
+ tool("computer_close_session", "End access and release mouse and keyboard input.", session, ["session_id"]),
+];
+const failure = (code, message, extra = {}) => ({ ok: false, code, message, ...extra });
+const content = (state, image) => ({ content: [{ type: "text", text: JSON.stringify(state) }, ...(image ? [{ type: "image", mimeType: "image/png", data: image.toString("base64") }] : [])], isError: state.ok !== true });
+const digest = (image) => createHash("sha256").update(image).digest("hex");
+const sameBounds = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+function validateAction(action, size) {
+ if (!action || typeof action !== "object" || Array.isArray(action)) throw new Error("An action object is required.");
+ const point = (p) => { if (!p || !Number.isFinite(p.x) || !Number.isFinite(p.y) || p.x < 0 || p.y < 0 || p.x >= size.width || p.y >= size.height) throw new Error("The point must be inside the approved screenshot."); };
+ if (["click", "double_click", "scroll"].includes(action.type)) point(action);
+ else if (action.type === "drag") { if (!Array.isArray(action.path) || action.path.length < 2 || action.path.length > 64) throw new Error("A drag needs 2 to 64 points."); action.path.forEach(point); }
+ else if (action.type === "type") { if (typeof action.text !== "string" || action.text.length < 1 || action.text.length > 1000 || /[\u0000-\u0008\u000b\u000c\u000e-\u001f]/.test(action.text)) throw new Error("Text must contain 1 to 1000 printable characters."); }
+ else if (action.type === "key") { if (typeof action.key !== "string" || action.key.length > 20) throw new Error("A supported key name is required."); }
+ else throw new Error("This action is unavailable. Use click, double_click, drag, scroll, type, or key.");
+ if (action.type === "scroll" && (!Number.isInteger(action.delta) || Math.abs(action.delta) > 20 || action.delta === 0 || !["vertical", "horizontal", undefined].includes(action.axis))) throw new Error("Scroll needs 1 to 20 steps and a vertical or horizontal axis.");
+}
+export async function createPortableComputerUseHost({ profile, capture, preview }) {
+ const native = createNativeDesktop();
+ const availability = await native.ready;
+ const captureWindow = process.platform === "linux" ? (window) => native.call("capture", { window }) : capture;
+ const directory = path.join(profile, "computer-use");
+ await mkdir(directory, { recursive: true, mode: 0o700 });
+ const discoveryPath = path.join(directory, "connection.json");
+ const token = randomBytes(32).toString("hex");
+ const connections = new Map();
+ let lease = null, closed = false;
+ const state = () => [...connections.values()].flatMap((entry) => entry.session ? [{
+ connectionId: entry.id, id: entry.session.id, phase: entry.session.phase, appName: entry.session.window.appName,
+ task: entry.session.purpose, mode: entry.session.mode, windows: entry.session.windows.map((w) => ({ id: w.id, title: w.title })),
+ windowTitle: entry.session.window.title, status: entry.session.status, recoverable: false,
+ canContinue: entry.session.phase === "paused", previewVisible: entry.session.previewVisible,
+ remainingSeconds: Math.max(0, Math.ceil((entry.session.expiresAt - Date.now()) / 1000)),
+ }] : []);
+ const update = () => preview?.update(state());
+ async function stop(entry) {
+ const value = entry.session;
+ if (!value) return;
+ entry.session = null; value.observation = null; value.approval?.resolve(failure("approval_cancelled", "The request ended without access."));
+ if (lease === entry.id) lease = null;
+ await native.call("stop").catch(() => {});
+ update();
+ }
+ function pause(entry, message) {
+ const value = entry.session;
+ if (!value || value.phase === "approval") return;
+ value.phase = "paused"; value.observation = null; value.status = message;
+ void native.call("stop").catch(() => {}); update();
+ }
+ native.onInput(() => {
+ for (const entry of connections.values()) if (entry.session?.phase === "working") pause(entry, "Your input interrupted control. Choose Continue when you are ready.");
+ });
+ const server = createServer((socket) => {
+ if (closed || connections.size >= 16) { socket.destroy(); return; }
+ const entry = { id: randomUUID(), session: null, socket, busy: false };
+ connections.set(entry.id, entry);
+ let incoming = "", authenticated = false;
+ const authTimer = setTimeout(() => socket.destroy(), 5000);
+ socket.setEncoding("utf8");
+ const close = () => { clearTimeout(authTimer); connections.delete(entry.id); void stop(entry); };
+ socket.once("close", close); socket.on("error", () => socket.destroy());
+ const send = (message) => { if (!socket.destroyed) socket.write(JSON.stringify(message) + "\n"); };
+ const respond = async (message) => {
+ if (!message || message.jsonrpc !== "2.0" || !methods.has(message.method)) {
+ send({ jsonrpc: "2.0", id: message?.id ?? null, error: { code: -32601, message: "Method not available to the agent." } }); return;
+ }
+ if (message.method === "notifications/cancelled") { if (entry.session?.approvalRequest === message.params?.requestId) await stop(entry); return; }
+ if (message.method === "notifications/initialized") return;
+ let result;
+ try {
+ if (message.method === "initialize") result = { protocolVersion: "2025-11-25", capabilities: { tools: {} }, serverInfo: { name: "omnirush-computer-use", version: "0.1.0" }, instructions: "Use only the approved app window. Foreground control pauses on person input. Only the person can approve or Continue. Observe after every action; never retry partial input automatically. For Blender background work prefer its Python CLI or an app connector." };
+ else if (message.method === "ping") result = {};
+ else if (message.method === "tools/list") result = { tools };
+ else {
+ const name = message.params?.name, args = message.params?.arguments ?? {};
+ if (entry.busy && !["computer_session_status", "computer_close_session"].includes(name)) result = content(failure("busy", "Wait for the previous operation."));
+ else {
+ const ownsBusy = !["computer_session_status", "computer_close_session"].includes(name);
+ if (ownsBusy) entry.busy = true;
+ try { result = await call(entry, name, args, message.id); }
+ finally { if (ownsBusy) entry.busy = false; }
+ }
+ }
+ if (message.id !== undefined) send({ jsonrpc: "2.0", id: message.id, result });
+ } catch (error) { if (message.id !== undefined) send({ jsonrpc: "2.0", id: message.id, result: content(failure(error.code ?? "desktop_error", error.message)) }); }
+ };
+ socket.on("data", (chunk) => {
+ incoming += chunk;
+ if (incoming.length > 1_048_576) { socket.destroy(); return; }
+ let end;
+ while ((end = incoming.indexOf("\n")) >= 0) {
+ const line = incoming.slice(0, end); incoming = incoming.slice(end + 1);
+ let message; try { message = JSON.parse(line); } catch { socket.destroy(); return; }
+ if (!message || typeof message !== "object" || Array.isArray(message)) { socket.destroy(); return; }
+ if (!authenticated) {
+ const candidate = typeof message.token === "string" ? Buffer.from(message.token) : Buffer.alloc(0);
+ if (candidate.length !== token.length || !timingSafeEqual(candidate, Buffer.from(token))) { socket.destroy(); return; }
+ authenticated = true; clearTimeout(authTimer); continue;
+ }
+ void respond(message);
+ }
+ });
+ });
+ async function active(entry, id) {
+ const value = entry.session;
+ if (!value || value.id !== id || value.phase === "approval") return null;
+ if (Date.now() >= value.expiresAt) { await stop(entry); return null; }
+ if (Date.now() - value.lastOperation > 120_000 && value.phase === "working") pause(entry, "Paused after two minutes without work. Choose Continue when ready.");
+ return value;
+ }
+ async function observation(entry, value, includeImage) {
+ if (value.phase !== "working") return content(failure("session_paused", value.status, { next: "human_takeover" }));
+ const before = await native.call("state");
+ const window = await native.call("window", { id: value.window.id });
+ if (window.identity !== value.window.identity) { await stop(entry); return content(failure("window_unavailable", "The approved app closed or restarted. Start a new session.")); }
+ let captured = await captureWindow(window);
+ let stable = false;
+ for (let attempt = 0; attempt < 4; attempt++) {
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ const next = await captureWindow(window);
+ if (digest(next.image) === digest(captured.image)) { captured = next; stable = true; break; }
+ captured = next;
+ }
+ if (!stable) return content(failure("window_changing", "Wait for the app to finish updating, then observe again."));
+
+ const after = await native.call("state");
+ if (entry.session !== value || value.phase !== "working" || before.generation !== after.generation || !after.available) { pause(entry, "Desktop input or availability changed. Choose Continue when ready."); return content(failure("session_paused", "Observe after the person continues.")); }
+ const id = randomUUID();
+ value.window = window; value.lastOperation = Date.now();
+ value.observation = { id, time: Date.now(), generation: after.generation, window, size: captured.size, hash: digest(captured.image) };
+ preview?.image(captured.image);
+ update();
+ return content({ ok: true, session_id: value.id, observation_id: id, window_title: window.title, image_size: captured.size, window_bounds: window.bounds, coordinates: "screenshot pixels relative to the approved window", elements: [], modes, requires_visual_model: true }, includeImage ? captured.image : undefined);
+ }
+ async function call(entry, name, args, requestId) {
+ if (!tools.some((item) => item.name === name)) return content(failure("unknown_tool", "Unknown Computer Use tool."));
+ if (!availability.supported) return content(failure(availability.code ?? "desktop_unavailable", availability.error));
+ if (name === "computer_discover") {
+ const windows = await native.call("list");
+ const apps = new Map();
+ for (const window of windows) apps.set(window.pid, { app_id: window.appId, pid: window.pid, name: window.appName });
+ return content({ ok: true, modes, background_control: false, apps: [...apps.values()] });
+ }
+ if (name === "computer_open_session") {
+ if (!modes.includes(args.mode ?? "observe")) return content(failure("unsupported_mode", "This platform supports read-only observation or foreground mouse and keyboard control."));
+ if (typeof args.app_id !== "string" || typeof args.purpose !== "string" || !args.purpose.trim() || args.purpose.length > 500) return content(failure("invalid_request", "Name a running app and describe the task in up to 500 characters."));
+ if (lease) return content(failure("session_busy", "Another Computer Use request is active. End it first."));
+ lease = entry.id;
+ try {
+ const windows = (await native.call("list")).filter((window) => window.appId === args.app_id && (args.pid === undefined || window.pid === args.pid));
+ if (!windows.length) { lease = null; return content(failure("app_unavailable", "Open the app yourself, then discover its windows again.")); }
+ const value = { id: randomUUID(), window: windows[0], windows, mode: args.mode ?? "observe", purpose: args.purpose, phase: "approval", previewVisible: true, expiresAt: Date.now() + 900_000, lastOperation: Date.now(), attempts: 0, requests: new Set(), observation: null, approvalRequest: requestId };
+ entry.session = value;
+ const approval = new Promise((resolve) => { value.approval = { resolve }; });
+ const timeout = setTimeout(() => { if (entry.session === value) void stop(entry); }, 60_000);
+ update();
+ try {
+ const result = await approval;
+ if (result.ok !== true) return content(result);
+ return content({ ok: true, session_id: value.id, state: "active", app_id: value.window.appId, pid: value.window.pid, mode: value.mode, window_title: value.window.title, expires_in_seconds: 900, background_control: false });
+ } finally { clearTimeout(timeout); value.approval = null; }
+ } catch (error) { if (lease === entry.id) lease = null; throw error; }
+ }
+ const value = await active(entry, args.session_id);
+ if (!value) return content(failure("session_unavailable", "This session has ended or belongs to another client."));
+ if (name === "computer_close_session") { await stop(entry); return content({ ok: true }); }
+ if (name === "computer_session_status") return content({ ok: true, session_id: value.id, state: value.phase === "working" ? "active" : "paused", phase: value.phase, mode: value.mode, background_control: false, panel_visible: value.previewVisible, remaining_seconds: Math.max(0, Math.ceil((value.expiresAt - Date.now()) / 1000)), message: value.status });
+ if (name === "computer_observe") return observation(entry, value, args.include_image !== false);
+ if (value.mode !== "control") return content(failure("scope_denied", "This session is read-only."));
+ if (value.phase !== "working") return content(failure("session_paused", value.status, { next: "human_takeover" }));
+ if (++value.attempts > 200) { await stop(entry); return content(failure("action_limit", "Start a new session after 200 action attempts.")); }
+ const observed = value.observation; value.observation = null;
+ if (typeof args.request_id !== "string" || !args.request_id || args.request_id.length > 100 || value.requests.has(args.request_id)) return content(failure("duplicate_request", "Use a unique request ID. Observe the current state instead of repeating input."));
+ value.requests.add(args.request_id);
+ if (!observed || observed.id !== args.observation_id || Date.now() - observed.time > 15_000) return content(failure("stale_observation", "Observe the window again before acting.", { may_have_acted: false }));
+ try { validateAction(args.action, observed.size); } catch (error) { return content(failure("invalid_action", error.message, { may_have_acted: false })); }
+ const current = await native.call("window", { id: value.window.id });
+ if (current.identity !== observed.window.identity || !sameBounds(current.bounds, observed.window.bounds)) return content(failure("window_changed", "The window moved or changed. Observe again.", { may_have_acted: false }));
+ try { await native.call("check", { window: current, generation: observed.generation, deadline: value.expiresAt }); }
+ catch (error) { pause(entry, error.message); return content(failure(error.code ?? "input_interrupted", error.message, { may_have_acted: false, next: "human_takeover" })); }
+ const fresh = await captureWindow(current);
+ if (digest(fresh.image) !== observed.hash) return content(failure("stale_observation", "The window changed. Observe again.", { may_have_acted: false }));
+ if (entry.session !== value || value.phase !== "working") return content(failure("session_paused", "The session was interrupted.", { may_have_acted: false }));
+ const scale = (p) => ({ ...p, x: Math.min(current.bounds.width - 1, Math.floor(p.x * current.bounds.width / observed.size.width)), y: Math.min(current.bounds.height - 1, Math.floor(p.y * current.bounds.height / observed.size.height)) });
+ const action = args.action.type === "drag" ? { ...args.action, path: args.action.path.map(scale) } : ["click", "double_click", "scroll"].includes(args.action.type) ? scale(args.action) : args.action;
+ try {
+ await native.call("act", { window: current, generation: observed.generation, action, deadline: value.expiresAt });
+ value.lastOperation = Date.now();
+ return content({ ok: true, session_id: value.id, request_id: args.request_id, dispatched: true, next: "observe" });
+ } catch (error) {
+ pause(entry, error.message);
+ return content(failure(error.code ?? "input_interrupted", error.message, { may_have_acted: true, next: "human_takeover" }));
+ }
+ }
+ await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => resolve(undefined)); });
+ const address = server.address();
+ if (!address || typeof address === "string") throw new Error("Computer Use could not create its local connection.");
+ const endpoint = { version: 1, host: "127.0.0.1", port: address.port, token };
+ const temp = discoveryPath + "." + randomUUID();
+ await writeFile(temp, JSON.stringify(endpoint), { mode: 0o600, flag: "wx" });
+ await rename(temp, discoveryPath);
+ const expiry = setInterval(() => { for (const entry of connections.values()) if (entry.session && Date.now() >= entry.session.expiresAt) void stop(entry); }, 1000);
+ const host = {
+ discoveryPath, state, availability,
+ pause(connectionId) { const entry = connections.get(connectionId); if (entry) pause(entry, "You took over. Choose Continue in OmniRush.ai when ready."); },
+ async apps() { if (!availability.supported) return { ok: false, apps: [] }; const windows = await native.call("list"); return { ok: true, apps: [...new Set(windows.map((w) => w.appName))] }; },
+ async action(input) {
+ const entry = connections.get(input?.connectionId), value = entry?.session;
+ if (!value || value.id !== input.id || !["approve", "deny", "resume", "stop", "hide", "show"].includes(input.action)) throw new Error("This Computer Use request has ended.");
+ if (input.action === "stop" || input.action === "deny") return stop(entry);
+ if (input.action === "hide") { value.previewVisible = false; update(); return; }
+ if (input.action === "show") { value.previewVisible = true; update(); return; }
+ if ((input.action === "approve" && value.phase !== "approval") || (input.action === "resume" && value.phase !== "paused")) throw new Error("This action is unavailable.");
+ const selected = input.action === "approve" ? value.windows.find((w) => w.id === input.windowId) : value.window;
+ if (!selected) throw new Error("Choose a window.");
+ const current = await native.call("window", { id: selected.id });
+ if (current.identity !== selected.identity) { await stop(entry); throw new Error("The app restarted. Start a new session."); }
+ if (value.mode === "control") await native.call("focus", { window: current });
+ if (entry.session !== value) throw new Error("The request ended.");
+ value.window = current; value.phase = "working"; value.status = ""; value.lastOperation = Date.now(); value.observation = null;
+ if (input.action === "approve") value.expiresAt = Date.now() + 900_000;
+ value.approval?.resolve({ ok: true }); update();
+ },
+ async close() { if (closed) return; closed = true; clearInterval(expiry); for (const entry of connections.values()) { await stop(entry); entry.socket.destroy(); } server.close(); preview?.close(); await native.close(); await unlink(discoveryPath).catch(() => {}); },
+ };
+ preview?.bind(host);
+ return host;
+}
diff --git a/apps/desktop/electron/computer-use-preview-preload.cjs b/apps/desktop/electron/computer-use-preview-preload.cjs
new file mode 100644
index 00000000..3626720e
--- /dev/null
+++ b/apps/desktop/electron/computer-use-preview-preload.cjs
@@ -0,0 +1,6 @@
+const { contextBridge, ipcRenderer } = require("electron");
+contextBridge.exposeInMainWorld("computerPreview", {
+ action: (value) => { if (["stop", "hide", "pause"].includes(value)) ipcRenderer.send("omnirush-computer-use-preview-action", value); },
+ onState: (listener) => ipcRenderer.on("omnirush-computer-use-preview-state", (_event, value) => listener(value)),
+ onImage: (listener) => ipcRenderer.on("omnirush-computer-use-preview-image", (_event, value) => listener(value)),
+});
diff --git a/apps/desktop/electron/computer-use-preview-renderer.js b/apps/desktop/electron/computer-use-preview-renderer.js
new file mode 100644
index 00000000..ebaa41f4
--- /dev/null
+++ b/apps/desktop/electron/computer-use-preview-renderer.js
@@ -0,0 +1,8 @@
+for (const name of ["pause", "hide", "stop"]) document.getElementById(name).addEventListener("click", () => window.computerPreview.action(name));
+window.computerPreview.onState((value) => {
+ document.getElementById("app").textContent = value.appName;
+ document.getElementById("status").textContent = value.phase === "paused" ? value.status : value.mode === "observe" ? "Read only · no mouse or keyboard access" : "Keep this app in front. Your input pauses control.";
+ document.getElementById("time").textContent = Math.ceil(value.remainingSeconds / 60) + " min";
+ document.getElementById("pause").hidden = value.mode === "observe" || value.phase === "paused";
+});
+window.computerPreview.onImage((value) => { document.getElementById("preview").src = value; });
diff --git a/apps/desktop/electron/computer-use-preview.css b/apps/desktop/electron/computer-use-preview.css
new file mode 100644
index 00000000..837b6edc
--- /dev/null
+++ b/apps/desktop/electron/computer-use-preview.css
@@ -0,0 +1 @@
+body{margin:0;padding:12px;background:#18181b;color:#fafafa;font:13px system-ui,sans-serif}header,footer{display:flex;align-items:center;gap:8px}header{justify-content:space-between}p{font-size:12px;line-height:1.4}img{display:block;width:100%;height:150px;object-fit:contain;background:#27272a;border-radius:8px}footer{margin-top:12px}button{padding:6px 10px;background:#3f3f46;color:white;border:0;border-radius:6px;cursor:pointer}#stop{margin-left:auto;background:#b91c1c}button:focus-visible{outline:2px solid #93c5fd}#time{font-variant-numeric:tabular-nums}
diff --git a/apps/desktop/electron/computer-use-preview.html b/apps/desktop/electron/computer-use-preview.html
new file mode 100644
index 00000000..33183bca
--- /dev/null
+++ b/apps/desktop/electron/computer-use-preview.html
@@ -0,0 +1,3 @@
+<!doctype html>
+<html><head><meta charset="utf-8"><meta http-equiv="Content-Security-Policy" content="default-src 'none'; script-src 'self'; style-src 'self'; img-src data:"><title>Computer Use</title><link rel="stylesheet" href="computer-use-preview.css"></head>
+<body><header><strong id="app">Computer Use</strong><span id="time"></span></header><p id="status" role="status">Your approved window · foreground control</p><img id="preview" alt="Latest approved window observation"><footer><button id="pause">Take over</button><button id="hide">Hide</button><button id="stop">Stop</button></footer><script src="computer-use-preview-renderer.js"></script></body></html>
diff --git a/apps/desktop/electron/computer-use-preview.mjs b/apps/desktop/electron/computer-use-preview.mjs
new file mode 100644
index 00000000..79bccef3
--- /dev/null
+++ b/apps/desktop/electron/computer-use-preview.mjs
@@ -0,0 +1,40 @@
+import { BrowserWindow, ipcMain, desktopCapturer } from "electron";
+import { fileURLToPath } from "node:url";
+export async function captureComputerWindow(window) {
+ const limit = 1600, ratio = Math.min(1, limit / Math.max(window.bounds.width, window.bounds.height));
+ const sources = await desktopCapturer.getSources({ types: ["window"], thumbnailSize: { width: Math.round(window.bounds.width * ratio), height: Math.round(window.bounds.height * ratio) }, fetchWindowIcons: false });
+ const source = sources.find((item) => Number(item.id.split(":")[1]) === window.id);
+ if (!source || source.thumbnail.isEmpty()) throw Object.assign(new Error("This window could not be captured. Keep it visible; protected video and some GPU windows may not support capture."), { code: "capture_unavailable" });
+ const size = source.thumbnail.getSize();
+ if (size.width <= 0 || size.height <= 0 || Math.abs(size.width / size.height - window.bounds.width / window.bounds.height) > 0.04) throw Object.assign(new Error("The captured window size changed. Observe again."), { code: "window_changed" });
+ return { image: source.thumbnail.toPNG(), size };
+}
+export function createComputerPreview() {
+ let host, panel, current;
+ const channel = "omnirush-computer-use-preview-action";
+ const action = (_event, value) => {
+ if (!panel || _event.sender !== panel.webContents || _event.senderFrame !== panel.webContents.mainFrame || !["stop", "hide", "pause"].includes(value) || !current) return;
+ if (value === "pause") host.pause(current.connectionId);
+ else void host.action({ connectionId: current.connectionId, id: current.id, action: value }).catch(() => {});
+ };
+ ipcMain.on(channel, action);
+ const show = () => {
+ if (panel && !panel.isDestroyed()) { panel.showInactive(); return; }
+ panel = new BrowserWindow({ width: 330, height: 300, title: "OmniRush.ai · Computer Use", show: false, alwaysOnTop: true, resizable: true, minimizable: false, webPreferences: { preload: fileURLToPath(new URL("./computer-use-preview-preload.cjs", import.meta.url)), nodeIntegration: false, contextIsolation: true, sandbox: true, backgroundThrottling: false } });
+ panel.webContents.setWindowOpenHandler(() => ({ action: "deny" }));
+ panel.webContents.on("will-navigate", (event) => event.preventDefault());
+ panel.on("close", (event) => { if (!current) return; event.preventDefault(); void host.action({ connectionId: current.connectionId, id: current.id, action: "hide" }).catch(() => {}); });
+ panel.webContents.once("did-finish-load", () => { if (current?.previewVisible) { panel.webContents.send("omnirush-computer-use-preview-state", current); panel.showInactive(); } });
+ void panel.loadFile(fileURLToPath(new URL("./computer-use-preview.html", import.meta.url)));
+ };
+ return {
+ bind(value) { host = value; },
+ update(states) {
+ current = states.find((value) => value.phase !== "approval");
+ if (!current || !current.previewVisible) { panel?.hide(); return; }
+ show(); panel.webContents.send("omnirush-computer-use-preview-state", current);
+ },
+ image(image) { panel?.webContents.send("omnirush-computer-use-preview-image", "data:image/png;base64," + image.toString("base64")); },
+ close() { current = null; panel?.destroy(); panel = null; ipcMain.removeListener(channel, action); },
+ };
+}
diff --git a/apps/desktop/electron/computer-use-relay.mjs b/apps/desktop/electron/computer-use-relay.mjs
new file mode 100644
index 00000000..3a0fb752
--- /dev/null
+++ b/apps/desktop/electron/computer-use-relay.mjs
@@ -0,0 +1,20 @@
+// Bundled stdio relay. Reads fresh discovery on each launch so app restarts do
+// not leave stale ports in a workspace. Only the desktop host can approve access.
+import { readFile } from "node:fs/promises";
+import { createConnection } from "node:net";
+const file = process.argv[2];
+try {
+ if (!file) throw new Error("Missing desktop connection. Enable Computer Use in OmniRush.ai.");
+ const value = JSON.parse(await readFile(file, "utf8"));
+ if (value.version !== 1 || value.host !== "127.0.0.1" || !Number.isInteger(value.port) || value.port <= 0 || value.port > 65535 || !/^[a-f0-9]{64}$/.test(value.token)) throw new Error("Invalid desktop connection. Restart OmniRush.ai.");
+ const socket = createConnection({ host: value.host, port: value.port });
+ socket.setTimeout(5000, () => { process.stderr.write("OmniRush.ai desktop did not respond.\n"); socket.destroy(); });
+ socket.once("connect", () => {
+ socket.setTimeout(0); socket.write(JSON.stringify({ token: value.token }) + "\n");
+ process.stdin.pipe(socket); socket.pipe(process.stdout);
+ });
+ socket.on("error", () => { process.stderr.write("Open OmniRush.ai and enable Computer Use before reconnecting.\n"); process.exitCode = 1; process.stdin.destroy(); });
+ socket.on("close", () => { process.stdin.destroy(); });
+ process.stdin.on("error", () => socket.destroy());
+ process.stdout.on("error", () => socket.destroy());
+} catch (error) { process.stderr.write(error.message + "\n"); process.exitCode = 1; }
diff --git a/apps/desktop/electron/computer-use.mjs b/apps/desktop/electron/computer-use.mjs
index ae4fd5ad..c389d0bc 100644
--- a/apps/desktop/electron/computer-use.mjs
+++ b/apps/desktop/electron/computer-use.mjs
@@ -9,10 +9,16 @@ import { fileURLToPath } from "node:url";
import { app } from "electron";
import { createComputerUseHost } from "./computer-use-host.mjs";
+import { createPortableComputerUseHost } from "./computer-use-portable-host.mjs";
+import { captureComputerWindow, createComputerPreview } from "./computer-use-preview.mjs";
let hostPromise;
async function computerUseHost() {
if (!hostPromise) {
+ if (process.platform === "win32" || process.platform === "linux") {
+ hostPromise = createPortableComputerUseHost({ profile: app.getPath("userData"), capture: captureComputerWindow, preview: createComputerPreview() }).catch((error) => { hostPromise = null; throw error; });
+ return hostPromise;
+ }
const executable = resolveComputerUseExecutable();
if (!executable) throw new Error("Computer Use helper unavailable.");
hostPromise = createComputerUseHost({ profile: app.getPath("userData"), executable }).catch((error) => { hostPromise = null; throw error; });
@@ -20,7 +26,8 @@ async function computerUseHost() {
return hostPromise;
}
async function getComputerUseState() { return hostPromise ? (await hostPromise).state() : []; }
-async function computerUseAction(value) { (await computerUseHost()).action(value); }
+async function computerUseAction(value) { await (await computerUseHost()).action(value); }
+function getComputerUseMcpEnvironment() { return process.platform === "darwin" ? {} : { ELECTRON_RUN_AS_NODE: "1" }; }
app.on("before-quit", () => { hostPromise?.then((host) => host.close()).catch(() => {}); });
const __dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -41,7 +48,11 @@ function computerUseHelperAppPath() {
async function getComputerUseMcpCommand() {
if (process.platform !== "darwin") {
- throw new Error("Desktop Computer Use requires macOS 14 or later. Use the built-in browser for website tasks.");
+ const host = await computerUseHost();
+ if (!host.availability.supported) throw new Error(host.availability.error);
+ const relay = app.isPackaged ? path.join(process.resourcesPath, "computer-use", "relay.mjs") : path.join(__dirname, "computer-use-relay.mjs");
+ if (!existsSync(relay)) throw new Error("Computer Use is missing from this build. Reinstall OmniRush.ai.");
+ return [process.execPath, relay, host.discoveryPath];
}
const helperExecutable = resolveComputerUseExecutable();
if (helperExecutable) return [helperExecutable, "relay", (await computerUseHost()).socketPath];
@@ -90,7 +101,9 @@ function resolveComputerUseExecutable() {
async function checkComputerUsePermissions() {
if (process.platform !== "darwin") {
- return { ok: false, accessibility: false, screenRecording: false, supported: false, error: "Desktop Computer Use is available on macOS 14 or later. Use the built-in browser for website tasks." };
+ const host = await computerUseHost();
+ const available = host.availability;
+ return { ok: available.supported, accessibility: available.supported, screenRecording: available.supported, supported: available.supported, protocolVersion: "omnirush.computer-use/1", platform: process.platform, modes: ["observe", "control"], backgroundControl: false, ...(available.error ? { error: available.error } : {}) };
}
// Spawn binary --check → read JSON from stdout → exit. Always fresh.
const bin = resolveComputerUseExecutable();
@@ -128,7 +141,7 @@ function spawnCheckPermissions(bin) {
async function listRunningApps() {
// Spawn binary --list-apps → read JSON from stdout → exit. Needs no TCC
// permissions, so this works before Computer Use setup is complete.
- if (process.platform !== "darwin") return { ok: false, apps: [] };
+ if (process.platform !== "darwin") return (await computerUseHost()).apps();
const bin = resolveComputerUseExecutable();
if (!bin) return { ok: false, apps: [] };
return new Promise((resolve) => {
@@ -152,7 +165,7 @@ async function listRunningApps() {
let setupProcess = null;
async function openComputerUseSetupApp() {
- if (process.platform !== "darwin") throw new Error("Desktop Computer Use requires macOS 14 or later.");
+ if (process.platform !== "darwin") { await computerUseHost(); return; }
const bin = resolveComputerUseExecutable();
if (!bin) throw new Error("The Computer Use helper is unavailable. Rebuild or reinstall OmniRush.ai.");
// Keep the responsible application consistent with --check and the MCP
@@ -176,6 +189,7 @@ async function openComputerUseSetupApp() {
export {
getComputerUseState,
+ getComputerUseMcpEnvironment,
computerUseAction,
checkComputerUsePermissions,
getComputerUseMcpCommand,
diff --git a/apps/desktop/electron/main.mjs b/apps/desktop/electron/main.mjs
index e2cf54cf..2406e8b3 100644
--- a/apps/desktop/electron/main.mjs
+++ b/apps/desktop/electron/main.mjs
@@ -26,6 +26,7 @@ import { registerUpdaterIpc } from "./updater.mjs";
import {
checkComputerUsePermissions,
getComputerUseMcpCommand,
+ getComputerUseMcpEnvironment,
getComputerUseState,
computerUseAction,
listRunningApps,
@@ -2004,6 +2005,7 @@ const desktopCommandHandlers = {
"getComputerUseMcpCommand": async (event, ...args) => {
return getComputerUseMcpCommand();
},
+ "getComputerUseMcpEnvironment": async () => getComputerUseMcpEnvironment(),
"checkComputerUsePermissions": async (event, ...args) => {
// Read permissions in the same child-process context as setup.
return checkComputerUsePermissions();
diff --git a/apps/desktop/package.json b/apps/desktop/package.json
index 1df668ba..f59da19b 100644
--- a/apps/desktop/package.json
+++ b/apps/desktop/package.json
@@ -53,7 +53,8 @@
"node-pty": "npm:@lydell/[email protected]",
"undici": "6.28.0",
"yaml": "^2.9.0",
- "zod": "^4.3.6"
+ "zod": "^4.3.6",
+ "koffi": "2.16.2"
},
"devDependencies": {
"@electron/asar": "^3.4.1",
diff --git a/apps/desktop/scripts/prepare-runtime-node-modules.mjs b/apps/desktop/scripts/prepare-runtime-node-modules.mjs
index f9d41194..9e6a80b2 100644
--- a/apps/desktop/scripts/prepare-runtime-node-modules.mjs
+++ b/apps/desktop/scripts/prepare-runtime-node-modules.mjs
@@ -5,7 +5,7 @@ import { fileURLToPath, pathToFileURL } from "node:url";
const dirnameHere = dirname(fileURLToPath(import.meta.url));
const desktopRoot = resolve(dirnameHere, "..");
-const requiredRoots = [{
+const requiredRoots = [{ name: "koffi" }, {
name: "@modelcontextprotocol/sdk",
resolveTarget: "@modelcontextprotocol/sdk/validation/ajv",
}];
diff --git a/evals/packages/labs/fixtures/portable-computer-use-app.mjs b/evals/packages/labs/fixtures/portable-computer-use-app.mjs
new file mode 100644
index 00000000..a88b188f
--- /dev/null
+++ b/evals/packages/labs/fixtures/portable-computer-use-app.mjs
@@ -0,0 +1,30 @@
+// Disposable native app used as an OS-level witness, never a real user's app.
+import { app, BrowserWindow, screen, nativeImage } from "electron";
+import { createInterface } from "node:readline";
+let windows = [];
+app.setPath("userData", process.argv.at(-1));
+const ready = app.whenReady().then(async () => {
+ for (const [index, title] of ["Workspace window", "Other window"].entries()) {
+ const window = new BrowserWindow({ width: 620, height: 450, x: 50 + index * 700, y: 120, title, webPreferences: { nodeIntegration: false, contextIsolation: true } });
+ const html = '<title>' + title + '</title><style>body{margin:0;background:white;color:black;font:16px sans-serif}button{position:absolute;left:70px;top:70px;width:180px;height:55px}input{position:absolute;left:70px;top:160px;width:300px;height:40px;caret-color:transparent}#count{position:absolute;left:70px;top:230px}</style><button id="increment">Increment</button><input id="draft" aria-label="Fixture draft" value="Initial draft"><p id="count">0</p><script>window.counter=0;document.getElementById("increment").onclick=()=>{document.getElementById("count").textContent=String(++window.counter)}</script>';
+ await window.loadURL("data:text/html," + encodeURIComponent(html)); windows.push(window);
+ }
+});
+createInterface({ input: process.stdin }).on("line", async (line) => {
+ const request = JSON.parse(line);
+ try {
+ await ready; let result;
+ if (request.method === "state") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript('({ count: window.counter, draft: document.getElementById("draft").value })')));
+ else if (request.method === "bounds") result = windows.map((w) => ({ bounds: w.getBounds(), contentBounds: w.getContentBounds(), scale: screen.getDisplayMatching(w.getBounds()).scaleFactor }));
+ else if (request.method === "focus_other") { windows[1].focus(); result = {}; }
+ else if (request.method === "cover") { windows[1].setBounds(windows[0].getBounds()); windows[1].setAlwaysOnTop(true); await windows[1].webContents.executeJavaScript('document.body.style.background="#ff00ff"'); windows[1].focus(); result = {}; }
+ else if (request.method === "image_pixel") { const image = nativeImage.createFromBuffer(Buffer.from(request.params.data, "base64")); const size = image.getSize(), pixels = image.toBitmap(); const offset = (Math.floor(size.height / 2) * size.width + Math.floor(size.width / 2)) * 4; result = { width: size.width, height: size.height, rgb: [pixels[offset + 2], pixels[offset + 1], pixels[offset]] }; }
+ else if (request.method === "change") { await windows[0].webContents.executeJavaScript('document.getElementById("draft").value="Changed by person"; document.body.style.background="#eeeeee"'); result = {}; }
+ else if (request.method === "minimize") { windows[0].minimize(); result = {}; }
+ else if (request.method === "restore") { windows[0].restore(); result = {}; }
+ else if (request.method === "close") { app.quit(); return; }
+ else throw new Error("Unknown fixture request.");
+ process.stdout.write(JSON.stringify({ id: request.id, result }) + "\n");
+ } catch (error) { process.stdout.write(JSON.stringify({ id: request.id, error: { message: error.message } }) + "\n"); }
+});
+process.stdin.on("end", () => app.quit());
diff --git a/evals/packages/labs/fixtures/portable-computer-use-host.mjs b/evals/packages/labs/fixtures/portable-computer-use-host.mjs
new file mode 100644
index 00000000..ca645968
--- /dev/null
+++ b/evals/packages/labs/fixtures/portable-computer-use-host.mjs
@@ -0,0 +1,27 @@
+// A minimal Electron host for the public desktop IPC boundary. It exercises the
+// product integration and native preview without accounts or inference services.
+import { app, BrowserWindow, desktopCapturer } from "electron";
+import { createInterface } from "node:readline";
+import { checkComputerUsePermissions, getComputerUseMcpCommand, getComputerUseMcpEnvironment, getComputerUseState, computerUseAction } from "../../../../apps/desktop/electron/computer-use.mjs";
+app.setPath("userData", process.argv.at(-1));
+const ready = app.whenReady();
+createInterface({ input: process.stdin }).on("line", async (line) => {
+ const request = JSON.parse(line);
+ try {
+ await ready; let result;
+ if (request.method === "permissions") result = await checkComputerUsePermissions();
+ else if (request.method === "command") result = { command: await getComputerUseMcpCommand(), environment: getComputerUseMcpEnvironment() };
+ else if (request.method === "sources") result = (await desktopCapturer.getSources({ types: ["window"], thumbnailSize: { width: 620, height: 450 } })).map((source) => ({ id: source.id, name: source.name, size: source.thumbnail.getSize(), empty: source.thumbnail.isEmpty() }));
+ else if (request.method === "state") result = await getComputerUseState();
+ else if (request.method === "preview_action") {
+ const preview = BrowserWindow.getAllWindows().find((window) => window.getTitle().includes("Computer Use"));
+ if (!preview) throw new Error("Native preview unavailable.");
+ if (!["pause", "hide", "stop"].includes(request.params.action)) throw new Error("Invalid preview command.");
+ await preview.webContents.executeJavaScript("document.getElementById(" + JSON.stringify(request.params.action) + ").click()"); result = {};
+ }
+ else if (request.method === "action") { await computerUseAction(request.params); result = {}; }
+ else throw new Error("Unknown host witness operation.");
+ process.stdout.write(JSON.stringify({ id: request.id, result }) + "\n");
+ } catch (error) { process.stdout.write(JSON.stringify({ id: request.id, error: { message: error.message } }) + "\n"); }
+});
+process.stdin.on("end", () => app.quit());
diff --git a/evals/specs/computer-use-portable-window.e2e.test.ts b/evals/specs/computer-use-portable-window.e2e.test.ts
new file mode 100644
index 00000000..c25ccc80
--- /dev/null
+++ b/evals/specs/computer-use-portable-window.e2e.test.ts
@@ -0,0 +1,107 @@
+import { expect } from "vitest";
+import { spec } from "@omnirush/testkit";
+import { portableComputerUseWorld, toolState } from "../worlds/computer-use.ts";
+
+const test = spec.world(portableComputerUseWorld, { timeout: 180_000 });
+
+test("Windows and Linux X11 Computer Use keeps native input within the approved window", async ({ world, step }) => {
+ await step("Desktop readiness exposes supported modes and discovery reveals no window contents", async () => {
+ expect(world.permissions).toMatchObject({ ok: true, supported: true, modes: ["observe", "control"], backgroundControl: false });
+ const discovered = toolState(await world.call("computer_discover"));
+ expect(discovered.background_control).toBe(false);
+ expect(JSON.stringify(discovered)).not.toContain("Initial draft");
+ expect(toolState(await world.call("computer_observe", { session_id: "unapproved" })).code).toBe("session_unavailable");
+ expect(toolState(await world.call("computer_open_session", { app_id: world.appId, mode: "assist", purpose: "Unsupported controls" })).code).toBe("unsupported_mode");
+ expect(await world.state()).toEqual([{ count: 0, draft: "Initial draft" }, { count: 0, draft: "Initial draft" }]);
+ });
+ const session = await step("Only a person can approve a particular native window", async () => {
+ const pending = world.call("computer_open_session", { app_id: world.appId, pid: world.pid, mode: "control", purpose: "Increment a disposable counter and edit a disposable draft." });
+ await expect.poll(async () => JSON.stringify(await world.hostState())).toContain('"phase":"approval"');
+ await expect(world.raw("omnirush/ui", { action: "approve" })).rejects.toThrow("Method not available to the agent");
+ expect(toolState(await world.peerCall("computer_open_session", { app_id: world.appId, mode: "observe", purpose: "Peer request" })).code).toBe("session_busy");
+ expect(await world.state()).toEqual([{ count: 0, draft: "Initial draft" }, { count: 0, draft: "Initial draft" }]);
+ await world.approve();
+ const opened = toolState(await pending);
+ expect(opened).toMatchObject({ ok: true, window_title: "Workspace window", mode: "control", background_control: false });
+ expect(typeof opened.session_id).toBe("string");
+ return opened.session_id;
+ });
+ await step("A second client cannot borrow the grant or read the approved window", async () => {
+ expect(toolState(await world.peerCall("computer_observe", { session_id: session })).code).toBe("session_unavailable");
+ expect(toolState(await world.peerCall("computer_session_status", { session_id: session })).code).toBe("session_unavailable");
+ });
+ await step("Real native mouse input increments only the selected window", async () => {
+ const observedReply = await world.call("computer_observe", { session_id: session });
+ expect(observedReply).toMatchObject({ content: expect.arrayContaining([expect.objectContaining({ type: "image", mimeType: "image/png" })]) });
+ const observed = toolState(observedReply);
+ expect(observed.ok, JSON.stringify(observed)).toBe(true);
+ const point = await world.point(observed, 150, 95);
+ const acted = toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "first-click", action: { type: "click", ...point } }));
+ expect(acted, JSON.stringify(acted)).toMatchObject({ ok: true, dispatched: true });
+ await expect.poll(() => world.state()).toEqual([{ count: 1, draft: "Initial draft" }, { count: 0, draft: "Initial draft" }]);
+ expect(toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "repeat-observation", action: { type: "click", ...point } })).code).toBe("stale_observation");
+ });
+ await step("Native keyboard input edits only the selected window", async () => {
+ const act = async (request: string, action: object) => {
+ const observed = toolState(await world.call("computer_observe", { session_id: session }));
+ expect(observed.ok, JSON.stringify(observed)).toBe(true);
+ const result = toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: request, action }));
+ expect(result, JSON.stringify(result)).toMatchObject({ ok: true, dispatched: true });
+ };
+ const observed = toolState(await world.call("computer_observe", { session_id: session }));
+ const point = await world.point(observed, 150, 180);
+ expect(toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "focus-draft", action: { type: "click", ...point } })).ok).toBe(true);
+ await act("select-draft", { type: "key", key: "a", modifiers: ["ctrl"] });
+ await act("write-draft", { type: "type", text: "OmniRush 123!" });
+ await expect.poll(() => world.state()).toEqual([{ count: 1, draft: "OmniRush 123!" }, { count: 0, draft: "Initial draft" }]);
+ });
+ await step("Changed pixels and out-of-window coordinates cannot dispatch input", async () => {
+ const observed = toolState(await world.call("computer_observe", { session_id: session }));
+ await world.change();
+ expect(toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "stale-click", action: { type: "click", x: 1, y: 1 } }))).toMatchObject({ ok: false, code: "stale_observation", may_have_acted: false });
+ const fresh = toolState(await world.call("computer_observe", { session_id: session }));
+ expect(toolState(await world.call("computer_act", { session_id: session, observation_id: fresh.observation_id, request_id: "outside-click", action: { type: "click", x: -1, y: 20 } }))).toMatchObject({ ok: false, code: "invalid_action", may_have_acted: false });
+ expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);
+ });
+ await step("Focus moving to another window stops keyboard input", async () => {
+ const observed = toolState(await world.call("computer_observe", { session_id: session }));
+ await world.focusOther();
+ const result = toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "wrong-focus", action: { type: "type", text: "Forbidden" } }));
+ expect(result, JSON.stringify(result)).toMatchObject({ ok: false, code: "focus_changed" });
+ expect(toolState(await world.call("computer_session_status", { session_id: session })).state).toBe("paused");
+ expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);
+ await world.approve("resume");
+ });
+ await step("Take over pauses control and only the person can continue", async () => {
+ const observed = toolState(await world.call("computer_observe", { session_id: session }));
+ await world.previewAction("pause");
+ await expect.poll(async () => toolState(await world.call("computer_session_status", { session_id: session })).state).toBe("paused");
+ expect(toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "paused-input", action: { type: "type", text: "Forbidden" } })).code).toBe("session_paused");
+ await expect(world.raw("omnirush/ui", { action: "resume" })).rejects.toThrow("Method not available to the agent");
+ await world.approve("resume");
+ expect(toolState(await world.call("computer_session_status", { session_id: session })).state).toBe("active");
+ expect(toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "invalidated-input", action: { type: "type", text: "Forbidden" } })).code).toBe("stale_observation");
+ });
+ await step("Stop in the native preview revokes access and leaves both windows intact", async () => {
+ await world.previewAction("stop");
+ expect(toolState(await world.call("computer_observe", { session_id: session })).code).toBe("session_unavailable");
+ expect(await world.hostState()).toEqual([]);
+ expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);
+ });
+ await step("Read-only approval never permits mouse or keyboard input", async () => {
+ const pending = world.call("computer_open_session", { app_id: world.appId, pid: world.pid, mode: "observe", purpose: "Read the disposable fixture." });
+ await world.approve();
+ const opened = toolState(await pending);
+ expect(opened.ok).toBe(true);
+ const observed = toolState(await world.call("computer_observe", { session_id: opened.session_id, include_image: false }));
+ expect(observed.ok).toBe(true);
+ expect(toolState(await world.call("computer_act", { session_id: opened.session_id, observation_id: observed.observation_id, request_id: "read-only-type", action: { type: "type", text: "Forbidden" } })).code).toBe("scope_denied");
+ await world.cover();
+ const covered = await world.call("computer_observe", { session_id: opened.session_id });
+ expect(toolState(covered).ok, JSON.stringify(toolState(covered))).toBe(true);
+ expect(await world.imagePixel(covered)).toMatchObject({ rgb: [238, 238, 238] });
+ await world.closeClient();
+ await expect.poll(() => world.hostState()).toEqual([]);
+ expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);
+ });
+});
diff --git a/evals/worlds/computer-use.ts b/evals/worlds/computer-use.ts
index 9dfdd597..cc2e7437 100644
--- a/evals/worlds/computer-use.ts
+++ b/evals/worlds/computer-use.ts
@@ -13,8 +13,8 @@ function record(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
-function pipeClient(executable: string, args: string[]) {
- const child = spawn(executable, args, { stdio: ["pipe", "pipe", "pipe"] });
+function pipeClient(executable: string, args: string[], environment: Record<string, string> = {}) {
+ const child = spawn(executable, args, { stdio: ["pipe", "pipe", "pipe"], env: { ...process.env, ...environment } });
let stderr = "";
child.stderr.on("data", (chunk: Buffer) => { stderr = (stderr + chunk.toString()).slice(-2000); });
const pending = new Map<number, { resolve(value: unknown): void; reject(error: Error): void; timer: NodeJS.Timeout }>();
@@ -242,3 +242,92 @@ export async function computerUseWorld(_seed: Seed, { place }: { place: Place })
};
} catch (error) { await close(); throw error; }
}
+
+/** Desktop-native Windows/X11 journey with disposable Electron windows. */
+export async function portableComputerUseWorld(_seed: Seed, { place }: { place: Place }) {
+ if (place.kind !== "local" || !["win32", "linux"].includes(process.platform)) throw new SkipError("Windows or Linux X11 graphical desktop placement");
+ if (process.platform === "linux") {
+ if (process.env.WAYLAND_DISPLAY || process.env.XDG_SESSION_TYPE === "wayland") throw new SkipError("Linux X11; Wayland portal control is not implemented");
+ if (!process.env.DISPLAY) throw new SkipError("Linux graphical display");
+ }
+ const root = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
+ const requireDesktop = createRequire(join(root, "apps/desktop/package.json"));
+ const executable = requireDesktop("electron");
+ if (typeof executable !== "string") throw new Error("Electron executable unavailable.");
+ const directory = await mkdtemp(join(tmpdir(), "omnirush-native-journey-"));
+ const launchFlags = process.platform === "linux" ? ["--no-sandbox"] : [];
+ const fixture = pipeClient(executable, [...launchFlags, join(root, "evals/packages/labs/fixtures/portable-computer-use-app.mjs"), join(directory, "fixture")]);
+ const controller = pipeClient(executable, [...launchFlags, join(root, "evals/packages/labs/fixtures/portable-computer-use-host.mjs"), join(directory, "host")]);
+ let client: ReturnType<typeof pipeClient> | undefined;
+ let peer: ReturnType<typeof pipeClient> | undefined;
+ try {
+ await fixture.request("state");
+ const permissions = await controller.request("permissions", {}, 20_000);
+ if (!record(permissions) || permissions.ok !== true) throw new Error("Desktop readiness failed: " + JSON.stringify(permissions));
+ const launch = await controller.request("command");
+ if (!record(launch) || !Array.isArray(launch.command) || !launch.command.every((v): v is string => typeof v === "string") || !record(launch.environment)) throw new Error("Missing desktop MCP command.");
+ const environment: Record<string, string> = {};
+ for (const [key, value] of Object.entries(launch.environment)) { if (typeof value !== "string") throw new Error("Invalid MCP environment."); environment[key] = value; }
+ const command = launch.command;
+ client = pipeClient(command[0], command.slice(1), environment);
+ peer = pipeClient(command[0], command.slice(1), environment);
+ await client.request("initialize", { protocolVersion: "2025-11-25", clientInfo: { name: "native-journey", version: "1" }, capabilities: {} });
+ await peer.request("initialize", { protocolVersion: "2025-11-25", clientInfo: { name: "native-peer", version: "1" }, capabilities: {} });
+ const discovery = toolState(await client.request("tools/call", { name: "computer_discover", arguments: {} }));
+ if (!Array.isArray(discovery.apps)) throw new Error("No app discovery.");
+ const appInfo = discovery.apps.find((value: unknown) => record(value) && value.pid === fixture.pid);
+ if (!record(appInfo) || typeof appInfo.app_id !== "string") throw new Error("Fixture was not discoverable: " + JSON.stringify(discovery));
+ const main = client, other = peer;
+ return {
+ permissions,
+ appId: appInfo.app_id, pid: fixture.pid,
+ call: (name: string, args: Record<string, unknown> = {}) => main.request("tools/call", { name, arguments: args }, 70_000),
+ peerCall: (name: string, args: Record<string, unknown> = {}) => other.request("tools/call", { name, arguments: args }),
+ raw: (method: string, params: Record<string, unknown> = {}) => main.request(method, params),
+ hostState: () => controller.request("state"),
+ state: () => fixture.request("state"),
+ change: () => fixture.request("change"),
+ focusOther: () => fixture.request("focus_other"),
+ cover: () => fixture.request("cover"),
+ imagePixel(reply: unknown) {
+ if (!record(reply) || !Array.isArray(reply.content)) throw new Error("Missing image content.");
+ const image = reply.content.find((item: unknown) => record(item) && item.type === "image");
+ if (!record(image) || typeof image.data !== "string") throw new Error("Missing image data.");
+ return fixture.request("image_pixel", { data: image.data });
+ },
+ minimize: () => fixture.request("minimize"),
+ restore: () => fixture.request("restore"),
+ cancel: () => main.cancelPending(),
+ previewAction: (action: string) => controller.request("preview_action", { action }),
+ async approve(action = "approve") {
+ const deadline = Date.now() + 15_000;
+ for (;;) {
+ const states = await controller.request("state");
+ if (Array.isArray(states)) {
+ const state = states.find((value: unknown) => record(value) && (action === "approve" ? value.phase === "approval" : true));
+ if (record(state) && Array.isArray(state.windows)) {
+ const window = state.windows.find((value: unknown) => record(value) && value.title === "Workspace window");
+ if (!record(window) || typeof window.id !== "number") throw new Error("No workspace window in approval.");
+ await controller.request("action", { connectionId: state.connectionId, id: state.id, action, windowId: window.id });
+ return;
+ }
+ }
+ if (Date.now() >= deadline) throw new Error("Timed out waiting for native consent.");
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ }
+ },
+ async point(observation: Record<string, unknown>, x: number, y: number) {
+ const bounds = await fixture.request("bounds");
+ if (!Array.isArray(bounds) || !record(bounds[0]) || !record(bounds[0].contentBounds) || typeof bounds[0].scale !== "number") throw new Error("Missing fixture bounds.");
+ const native = observation.window_bounds, image = observation.image_size, contentBounds = bounds[0].contentBounds;
+ if (!record(native) || !record(image) || typeof native.x !== "number" || typeof native.y !== "number" || typeof native.width !== "number" || typeof native.height !== "number" || typeof image.width !== "number" || typeof image.height !== "number" || typeof contentBounds.x !== "number" || typeof contentBounds.y !== "number") throw new Error("Missing image bounds.");
+ return { x: ((contentBounds.x + x) * bounds[0].scale - native.x) * image.width / native.width, y: ((contentBounds.y + y) * bounds[0].scale - native.y) * image.height / native.height };
+ },
+ async closeClient() { await main.close(); },
+ async [Symbol.asyncDispose]() { await Promise.all([main.close(), other.close(), controller.close(), fixture.close()]); await rm(directory, { recursive: true, force: true }); },
+ };
+ } catch (error) {
+ await Promise.all([client?.close(), peer?.close(), controller.close(), fixture.close()]);
+ await rm(directory, { recursive: true, force: true }); throw error;
+ }
+}
diff --git a/packages/types/src/desktop-ipc.ts b/packages/types/src/desktop-ipc.ts
index dd349a73..dfe86760 100644
--- a/packages/types/src/desktop-ipc.ts
+++ b/packages/types/src/desktop-ipc.ts
@@ -448,6 +448,9 @@ export type UiControlBridgeInfo = {
};
export type ComputerUsePermissions = {
+ platform?: string;
+ modes?: string[];
+ backgroundControl?: boolean;
ok: boolean;
accessibility: boolean;
screenRecording: boolean;
@@ -544,6 +547,7 @@ export type DesktopCommandMap = {
getUiControlBridgeInfo: { args: []; result: UiControlBridgeInfo | null };
getOmniRushUiMcpCommand: { args: []; result: string[] };
getComputerUseMcpCommand: { args: []; result: string[] };
+ getComputerUseMcpEnvironment: { args: []; result: Record<string, string> };
getComputerUseState: { args: []; result: unknown };
computerUseAction: { args: [value: { connectionId: string; id: string; action: string; windowId?: number }]; result: void };
getOmniRushUiMcpEnvironment: { args: []; result: Record<string, string> };
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index ba4c6970..11d98d2d 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -382,6 +382,9 @@ importers:
jsonc-parser:
specifier: ^3.2.1
version: 3.3.1
+ koffi:
+ specifier: 2.16.2
+ version: 2.16.2
minimatch:
specifier: ^10.2.5
version: 10.2.5
@@ -7245,6 +7248,9 @@ packages:
resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==}
engines: {node: '>=6'}
+ resolution: {integrity: sha512-owU0MRwv6xkrVqCd+33uw6BaYppkTRXbO/rVdJNI2dvZG0gzyRhYwW25eWtc5pauwK8TGh3AbkFONSezdykfSA==}
+
resolution: {integrity: sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q==}
@@ -15794,6 +15800,8 @@ snapshots:
+