Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

test: exercise native Computer Use through main-app setup

Task Builderfc3934781a3a
diff --git a/.github/workflows/computer-use-native.yml b/.github/workflows/computer-use-native.yml
index 2a115628..739a0e39 100644
--- a/.github/workflows/computer-use-native.yml
+++ b/.github/workflows/computer-use-native.yml
@@ -21,7 +21,7 @@ jobs:
matrix:
os: [windows-latest, ubuntu-latest]
runs-on: ${{ matrix.os }}
- timeout-minutes: 20
+ timeout-minutes: 30
steps:
- uses: actions/checkout@v6
with:
@@ -32,6 +32,9 @@ jobs:
- uses: pnpm/action-setup@v4
with:
version: 11.4.0
+ - uses: oven-sh/setup-bun@v2
+ with:
+ bun-version: 1.4.2
- name: Install app and journey dependencies
shell: bash
run: |
diff --git a/apps/desktop/electron/computer-use-portable-host.mjs b/apps/desktop/electron/computer-use-portable-host.mjs
index 3469a07e..32e6b3ab 100644
--- a/apps/desktop/electron/computer-use-portable-host.mjs
+++ b/apps/desktop/electron/computer-use-portable-host.mjs
@@ -217,7 +217,7 @@ export async function createPortableComputerUseHost({ profile, capture, preview
const temp = discoveryPath + "." + randomUUID();
await writeFile(temp, JSON.stringify(endpoint), { mode: 0o600, flag: "wx" });
await rename(temp, discoveryPath);
- const expiry = setInterval(() => { for (const entry of connections.values()) if (entry.session && Date.now() >= entry.session.expiresAt) void stop(entry); }, 1000);
+ const expiry = setInterval(() => { for (const entry of connections.values()) { const value = entry.session; if (!value) continue; if (Date.now() >= value.expiresAt) void stop(entry); else if (value.phase === "working" && Date.now() - value.lastOperation > 120_000) pause(entry, "Paused after two minutes without work. Choose Continue when ready."); } update(); }, 1000);
const host = {
discoveryPath, state, availability,
pause(connectionId) { const entry = connections.get(connectionId); if (entry) pause(entry, "You took over. Choose Continue in OmniRush.ai when ready."); },
diff --git a/evals/specs/computer-use-portable-window.e2e.test.ts b/evals/specs/computer-use-portable-window.e2e.test.ts
index c25ccc80..49e78514 100644
--- a/evals/specs/computer-use-portable-window.e2e.test.ts
+++ b/evals/specs/computer-use-portable-window.e2e.test.ts
@@ -1,4 +1,6 @@
import { expect } from "vitest";
+import { createAndSelectWorkspace, evalIn, waitFor } from "@omnirush/behaviors";
+import { browserScript } from "@omnirush/testkit";
import { spec } from "@omnirush/testkit";
import { portableComputerUseWorld, toolState } from "../worlds/computer-use.ts";
@@ -105,3 +107,54 @@ test("Windows and Linux X11 Computer Use keeps native input within the approved
expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);
});
});
+
+
+test("Computer Use setup and approval work from the main app on Windows and Linux X11", async ({ world, step }) => {
+ await using app = await world.desktop();
+ const { workspaceId } = await createAndSelectWorkspace(app, { path: world.workspacePath });
+ await step("Setup explains foreground control and offers the supported modes", async () => {
+ await evalIn(app, browserScript((value) => (location.hash = value), [`#/workspace/${workspaceId}/extensions/computer-use`]));
+ await waitFor(app, () => document.body.innerText.includes("Permissions are ready. Enable Computer Use for this workspace."), { timeoutMs: 60_000 });
+ expect(await evalIn(app, () => document.body.innerText.includes("Work through accessible controls"))).toBe(false);
+ expect(await evalIn(app, () => document.body.innerText.includes("foreground"))).toBe(true);
+ await evalIn(app, () => [...document.querySelectorAll("button")].find((button) => button.textContent.trim() === "Enable Computer Use")?.click());
+ await waitFor(app, () => document.body.innerText.includes("Ready · app access is approved when a session starts"), { timeoutMs: 60_000 });
+ });
+ const command = await evalIn(app, () => window.__OMNIRUSH_ELECTRON__.invokeDesktop("getComputerUseMcpCommand"));
+ const environment = await evalIn(app, () => window.__OMNIRUSH_ELECTRON__.invokeDesktop("getComputerUseMcpEnvironment"));
+ await step("Copy CLI connection produces a usable local command without the discovery secret", async () => {
+ await evalIn(app, () => [...document.querySelectorAll("button")].find((button) => button.textContent.trim() === "Copy CLI connection")?.click());
+ await waitFor(app, () => document.body.innerText.includes("CLI connection copied"));
+ const copied = await evalIn(app, async () => navigator.clipboard.readText(), { awaitPromise: true });
+ expect(typeof copied).toBe("string");
+ const config = JSON.parse(String(copied));
+ expect(config).toEqual({ mcpServers: { "computer-use": { command: Array.isArray(command) ? command[0] : "", args: Array.isArray(command) ? command.slice(1) : [], env: environment } } });
+ expect(copied).not.toContain('"token"');
+ });
+ await using client = await world.hostedClient(command, environment);
+ const session = await step("The app's window picker grants the selected native window", async () => {
+ const pending = client.call("computer_open_session", { app_id: world.appId, pid: world.pid, mode: "control", purpose: "Test the main app approval with a disposable window." });
+ await waitFor(app, () => Boolean(document.querySelector('select[aria-label="Window to allow"]')));
+ await evalIn(app, () => {
+ const picker = document.querySelector('select[aria-label="Window to allow"]');
+ if (!(picker instanceof HTMLSelectElement)) throw new Error("Missing window picker");
+ const option = [...picker.options].find((option) => option.textContent === "Workspace window");
+ if (!option) throw new Error("Missing fixture window");
+ picker.value = option.value; picker.dispatchEvent(new Event("change", { bubbles: true }));
+ [...document.querySelectorAll("button")].find((button) => button.textContent.trim() === "Allow and start")?.click();
+ });
+ const result = toolState(await pending);
+ expect(result).toMatchObject({ ok: true, window_title: "Workspace window", mode: "control" });
+ return result.session_id;
+ });
+ await step("The main app can stop access without relying on the agent", async () => {
+ expect(toolState(await client.call("computer_observe", { session_id: session })).ok).toBe(true);
+ await evalIn(app, async () => {
+ const states = await window.__OMNIRUSH_ELECTRON__.invokeDesktop("getComputerUseState");
+ if (!Array.isArray(states) || !states[0]) throw new Error("No active desktop session");
+ await window.__OMNIRUSH_ELECTRON__.invokeDesktop("computerUseAction", { connectionId: states[0].connectionId, id: states[0].id, action: "stop" });
+ }, { awaitPromise: true });
+ expect(toolState(await client.call("computer_session_status", { session_id: session })).code).toBe("session_unavailable");
+ expect(await world.state()).toEqual([{ count: 0, draft: "Initial draft" }, { count: 0, draft: "Initial draft" }]);
+ });
+});
diff --git a/evals/worlds/computer-use.ts b/evals/worlds/computer-use.ts
index cc2e7437..268d691e 100644
--- a/evals/worlds/computer-use.ts
+++ b/evals/worlds/computer-use.ts
@@ -279,6 +279,16 @@ export async function portableComputerUseWorld(_seed: Seed, { place }: { place:
if (!record(appInfo) || typeof appInfo.app_id !== "string") throw new Error("Fixture was not discoverable: " + JSON.stringify(discovery));
const main = client, other = peer;
return {
+ desktop: () => desktop({ name: "native-computer-use-setup", host: place.host(), profileDir: join(directory, "desktop-profile"), env: { OPENCODE_DB: join(directory, "desktop.db"), OMNIRUSH_GATEWAY_URL: "http://127.0.0.1:1", COREPACK_ENABLE_DOWNLOAD_PROMPT: "0", ELECTRON_DISABLE_SANDBOX: "1", OMNIRUSH_ELECTRON_USE_MOCK_KEYCHAIN: "1" } }),
+ workspacePath: join(directory, "workspace"),
+ async hostedClient(command: unknown, environment: unknown) {
+ if (!Array.isArray(command) || !command.every((part): part is string => typeof part === "string") || !record(environment)) throw new Error("Missing hosted MCP command.");
+ const env: Record<string, string> = {};
+ for (const [key, value] of Object.entries(environment)) { if (typeof value !== "string") throw new Error("Invalid hosted environment."); env[key] = value; }
+ const hosted = pipeClient(command[0], command.slice(1), env);
+ await hosted.request("initialize", { protocolVersion: "2025-11-25", clientInfo: { name: "desktop-native-journey", version: "1" }, capabilities: {} });
+ return { call: (name: string, args: Record<string, unknown> = {}) => hosted.request("tools/call", { name, arguments: args }, 70_000), [Symbol.asyncDispose]: hosted.close };
+ },
permissions,
appId: appInfo.app_id, pid: fixture.pid,
call: (name: string, args: Record<string, unknown> = {}) => main.request("tools/call", { name, arguments: args }, 70_000),
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 81a96e5c..c61bae97 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -115,6 +115,7 @@ allowBuilds:
electron: true
electron-winstaller: true
esbuild: true
+ koffi: true
# effect -> msgpackr -> msgpackr-extract ships an optional native accelerator;
# msgpackr falls back to pure JS, so skip compiling it in CI and images.
msgpackr-extract: false