A desktop coding agent with free access to frontier models.
TypeScript
test: exercise native Computer Use through main-app setup
Task Builderfc3934781a3a
diff --git a/.github/workflows/computer-use-native.yml b/.github/workflows/computer-use-native.ymlindex 2a115628..739a0e39 100644--- a/.github/workflows/computer-use-native.yml+++ b/.github/workflows/computer-use-native.yml@@ -21,7 +21,7 @@ jobs:matrix:os: [windows-latest, ubuntu-latest]runs-on: ${{ matrix.os }}- timeout-minutes: 20+ timeout-minutes: 30steps:- uses: actions/checkout@v6with:@@ -32,6 +32,9 @@ jobs:- uses: pnpm/action-setup@v4with:version: 11.4.0+ - uses: oven-sh/setup-bun@v2+ with:+ bun-version: 1.4.2- name: Install app and journey dependenciesshell: bashrun: |diff --git a/apps/desktop/electron/computer-use-portable-host.mjs b/apps/desktop/electron/computer-use-portable-host.mjsindex 3469a07e..32e6b3ab 100644--- a/apps/desktop/electron/computer-use-portable-host.mjs+++ b/apps/desktop/electron/computer-use-portable-host.mjs@@ -217,7 +217,7 @@ export async function createPortableComputerUseHost({ profile, capture, previewconst temp = discoveryPath + "." + randomUUID();await writeFile(temp, JSON.stringify(endpoint), { mode: 0o600, flag: "wx" });await rename(temp, discoveryPath);- const expiry = setInterval(() => { for (const entry of connections.values()) if (entry.session && Date.now() >= entry.session.expiresAt) void stop(entry); }, 1000);+ const expiry = setInterval(() => { for (const entry of connections.values()) { const value = entry.session; if (!value) continue; if (Date.now() >= value.expiresAt) void stop(entry); else if (value.phase === "working" && Date.now() - value.lastOperation > 120_000) pause(entry, "Paused after two minutes without work. Choose Continue when ready."); } update(); }, 1000);const host = {discoveryPath, state, availability,pause(connectionId) { const entry = connections.get(connectionId); if (entry) pause(entry, "You took over. Choose Continue in OmniRush.ai when ready."); },diff --git a/evals/specs/computer-use-portable-window.e2e.test.ts b/evals/specs/computer-use-portable-window.e2e.test.tsindex c25ccc80..49e78514 100644--- a/evals/specs/computer-use-portable-window.e2e.test.ts+++ b/evals/specs/computer-use-portable-window.e2e.test.ts@@ -1,4 +1,6 @@import { expect } from "vitest";+import { createAndSelectWorkspace, evalIn, waitFor } from "@omnirush/behaviors";+import { browserScript } from "@omnirush/testkit";import { spec } from "@omnirush/testkit";import { portableComputerUseWorld, toolState } from "../worlds/computer-use.ts";@@ -105,3 +107,54 @@ test("Windows and Linux X11 Computer Use keeps native input within the approvedexpect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);});});+++test("Computer Use setup and approval work from the main app on Windows and Linux X11", async ({ world, step }) => {+ await using app = await world.desktop();+ const { workspaceId } = await createAndSelectWorkspace(app, { path: world.workspacePath });+ await step("Setup explains foreground control and offers the supported modes", async () => {+ await evalIn(app, browserScript((value) => (location.hash = value), [`#/workspace/${workspaceId}/extensions/computer-use`]));+ await waitFor(app, () => document.body.innerText.includes("Permissions are ready. Enable Computer Use for this workspace."), { timeoutMs: 60_000 });+ expect(await evalIn(app, () => document.body.innerText.includes("Work through accessible controls"))).toBe(false);+ expect(await evalIn(app, () => document.body.innerText.includes("foreground"))).toBe(true);+ await evalIn(app, () => [...document.querySelectorAll("button")].find((button) => button.textContent.trim() === "Enable Computer Use")?.click());+ await waitFor(app, () => document.body.innerText.includes("Ready · app access is approved when a session starts"), { timeoutMs: 60_000 });+ });+ const command = await evalIn(app, () => window.__OMNIRUSH_ELECTRON__.invokeDesktop("getComputerUseMcpCommand"));+ const environment = await evalIn(app, () => window.__OMNIRUSH_ELECTRON__.invokeDesktop("getComputerUseMcpEnvironment"));+ await step("Copy CLI connection produces a usable local command without the discovery secret", async () => {+ await evalIn(app, () => [...document.querySelectorAll("button")].find((button) => button.textContent.trim() === "Copy CLI connection")?.click());+ await waitFor(app, () => document.body.innerText.includes("CLI connection copied"));+ const copied = await evalIn(app, async () => navigator.clipboard.readText(), { awaitPromise: true });+ expect(typeof copied).toBe("string");+ const config = JSON.parse(String(copied));+ expect(config).toEqual({ mcpServers: { "computer-use": { command: Array.isArray(command) ? command[0] : "", args: Array.isArray(command) ? command.slice(1) : [], env: environment } } });+ expect(copied).not.toContain('"token"');+ });+ await using client = await world.hostedClient(command, environment);+ const session = await step("The app's window picker grants the selected native window", async () => {+ const pending = client.call("computer_open_session", { app_id: world.appId, pid: world.pid, mode: "control", purpose: "Test the main app approval with a disposable window." });+ await waitFor(app, () => Boolean(document.querySelector('select[aria-label="Window to allow"]')));+ await evalIn(app, () => {+ const picker = document.querySelector('select[aria-label="Window to allow"]');+ if (!(picker instanceof HTMLSelectElement)) throw new Error("Missing window picker");+ const option = [...picker.options].find((option) => option.textContent === "Workspace window");+ if (!option) throw new Error("Missing fixture window");+ picker.value = option.value; picker.dispatchEvent(new Event("change", { bubbles: true }));+ [...document.querySelectorAll("button")].find((button) => button.textContent.trim() === "Allow and start")?.click();+ });+ const result = toolState(await pending);+ expect(result).toMatchObject({ ok: true, window_title: "Workspace window", mode: "control" });+ return result.session_id;+ });+ await step("The main app can stop access without relying on the agent", async () => {+ expect(toolState(await client.call("computer_observe", { session_id: session })).ok).toBe(true);+ await evalIn(app, async () => {+ const states = await window.__OMNIRUSH_ELECTRON__.invokeDesktop("getComputerUseState");+ if (!Array.isArray(states) || !states[0]) throw new Error("No active desktop session");+ await window.__OMNIRUSH_ELECTRON__.invokeDesktop("computerUseAction", { connectionId: states[0].connectionId, id: states[0].id, action: "stop" });+ }, { awaitPromise: true });+ expect(toolState(await client.call("computer_session_status", { session_id: session })).code).toBe("session_unavailable");+ expect(await world.state()).toEqual([{ count: 0, draft: "Initial draft" }, { count: 0, draft: "Initial draft" }]);+ });+});diff --git a/evals/worlds/computer-use.ts b/evals/worlds/computer-use.tsindex cc2e7437..268d691e 100644--- a/evals/worlds/computer-use.ts+++ b/evals/worlds/computer-use.ts@@ -279,6 +279,16 @@ export async function portableComputerUseWorld(_seed: Seed, { place }: { place:if (!record(appInfo) || typeof appInfo.app_id !== "string") throw new Error("Fixture was not discoverable: " + JSON.stringify(discovery));const main = client, other = peer;return {+ desktop: () => desktop({ name: "native-computer-use-setup", host: place.host(), profileDir: join(directory, "desktop-profile"), env: { OPENCODE_DB: join(directory, "desktop.db"), OMNIRUSH_GATEWAY_URL: "http://127.0.0.1:1", COREPACK_ENABLE_DOWNLOAD_PROMPT: "0", ELECTRON_DISABLE_SANDBOX: "1", OMNIRUSH_ELECTRON_USE_MOCK_KEYCHAIN: "1" } }),+ workspacePath: join(directory, "workspace"),+ async hostedClient(command: unknown, environment: unknown) {+ if (!Array.isArray(command) || !command.every((part): part is string => typeof part === "string") || !record(environment)) throw new Error("Missing hosted MCP command.");+ const env: Record<string, string> = {};+ for (const [key, value] of Object.entries(environment)) { if (typeof value !== "string") throw new Error("Invalid hosted environment."); env[key] = value; }+ const hosted = pipeClient(command[0], command.slice(1), env);+ await hosted.request("initialize", { protocolVersion: "2025-11-25", clientInfo: { name: "desktop-native-journey", version: "1" }, capabilities: {} });+ return { call: (name: string, args: Record<string, unknown> = {}) => hosted.request("tools/call", { name, arguments: args }, 70_000), [Symbol.asyncDispose]: hosted.close };+ },permissions,appId: appInfo.app_id, pid: fixture.pid,call: (name: string, args: Record<string, unknown> = {}) => main.request("tools/call", { name, arguments: args }, 70_000),diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex 81a96e5c..c61bae97 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -115,6 +115,7 @@ allowBuilds:electron: trueelectron-winstaller: trueesbuild: true+ koffi: true# effect -> msgpackr -> msgpackr-extract ships an optional native accelerator;# msgpackr falls back to pure JS, so skip compiling it in CI and images.msgpackr-extract: false