A desktop coding agent with free access to frontier models.
TypeScript
test: verify outside input and Stop using portable witnesses
Task Builder28292087577d
diff --git a/evals/packages/labs/fixtures/portable-computer-use-app.mjs b/evals/packages/labs/fixtures/portable-computer-use-app.mjsindex 96dc937b..041423a8 100644--- a/evals/packages/labs/fixtures/portable-computer-use-app.mjs+++ b/evals/packages/labs/fixtures/portable-computer-use-app.mjs@@ -1,6 +1,11 @@// Disposable native app used as an OS-level witness, never a real user's app.import { app, BrowserWindow, screen, nativeImage } from "electron";import { createInterface } from "node:readline";+import { connect } from "node:net";+const channel = connect({ host: "127.0.0.1", port: Number(process.env.OMNIRUSH_FIXTURE_PORT) });+channel.on("connect", () => channel.write(JSON.stringify({ token: process.env.OMNIRUSH_FIXTURE_TOKEN }) + "\n"));+import { createRequire } from "node:module";+const requireDesktop = createRequire(new URL("../../../../apps/desktop/package.json", import.meta.url));let windows = [];app.setPath("userData", process.argv.at(-1));const ready = app.whenReady().then(async () => {@@ -10,13 +15,34 @@ const ready = app.whenReady().then(async () => {await window.loadURL("data:text/html," + encodeURIComponent(html)); windows.push(window);}});-createInterface({ input: process.stdin }).on("line", async (line) => {+createInterface({ input: channel }).on("line", async (line) => {const request = JSON.parse(line);try {await ready; let result;if (request.method === "state") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript('({ count: window.counter, draft: document.getElementById("draft").value })')));else if (request.method === "bounds") result = windows.map((w) => ({ bounds: w.getBounds(), contentBounds: w.getContentBounds(), scale: screen.getDisplayMatching(w.getBounds()).scaleFactor }));else if (request.method === "focus_other") { windows[1].focus(); result = {}; }+ else if (request.method === "outside_input") {+ const koffi = requireDesktop("koffi");+ if (process.platform === "win32") {+ const keyboard = koffi.struct({ wVk: "uint16_t", wScan: "uint16_t", dwFlags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t" });+ const mouse = koffi.struct({ dx: "int32_t", dy: "int32_t", mouseData: "uint32_t", dwFlags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t" });+ const input = koffi.struct({ type: "uint32_t", value: koffi.union({ ki: keyboard, mi: mouse }) });+ const send = koffi.load("user32.dll").func("uint32_t __stdcall SendInput(uint32_t count, void *input, int size)");+ const data = Buffer.alloc(koffi.sizeof(input) * 2);+ koffi.encode(data, input, [{ type: 1, value: { ki: { wVk: 16, wScan: 0, dwFlags: 0, time: 0, dwExtraInfo: 0 } } }, { type: 1, value: { ki: { wVk: 16, wScan: 0, dwFlags: 2, time: 0, dwExtraInfo: 0 } } }], 2);+ if (send(2, data, koffi.sizeof(input)) !== 2) throw new Error("Outside keyboard stimulus unavailable.");+ } else {+ const x = koffi.load("libX11.so.6"), xt = koffi.load("libXtst.so.6"), display = x.func("void *XOpenDisplay(str name)")(null);+ if (!display) throw new Error("Outside input display unavailable.");+ try {+ const code = x.func("uint8_t XKeysymToKeycode(void *display, ulong symbol)")(display, 0xffe1);+ const send = xt.func("int XTestFakeKeyEvent(void *display, uint code, int down, ulong delay)");+ send(display, code, 1, 0); send(display, code, 0, 0); x.func("int XSync(void *display, int discard)")(display, 0);+ } finally { x.func("int XCloseDisplay(void *display)")(display); }+ }+ result = {};+ }else if (request.method === "cover") { windows[1].setBounds(windows[0].getBounds()); windows[1].setAlwaysOnTop(true); await windows[1].webContents.executeJavaScript('document.body.style.background="#ff00ff"'); windows[1].focus(); result = {}; }else if (request.method === "image_pixel") { const image = nativeImage.createFromBuffer(Buffer.from(request.params.data, "base64")); const size = image.getSize(), pixels = image.toBitmap(); const offset = ((size.height - 50) * size.width + 20) * 4; result = { width: size.width, height: size.height, rgb: [pixels[offset + 2], pixels[offset + 1], pixels[offset]] }; }else if (request.method === "change") { await windows[0].webContents.executeJavaScript('document.getElementById("draft").value="Changed by person"; document.body.style.background="#eeeeee"; new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))'); result = {}; }@@ -24,7 +50,7 @@ createInterface({ input: process.stdin }).on("line", async (line) => {else if (request.method === "restore") { windows[0].restore(); result = {}; }else if (request.method === "close") { app.quit(); return; }else throw new Error("Unknown fixture request.");- process.stdout.write(JSON.stringify({ id: request.id, result }) + "\n");- } catch (error) { process.stdout.write(JSON.stringify({ id: request.id, error: { message: error.message } }) + "\n"); }+ channel.write(JSON.stringify({ id: request.id, result }) + "\n");+ } catch (error) { channel.write(JSON.stringify({ id: request.id, error: { message: error.message } }) + "\n"); }});-process.stdin.on("end", () => app.quit());+channel.on("close", () => app.quit());diff --git a/evals/packages/labs/fixtures/portable-computer-use-host.mjs b/evals/packages/labs/fixtures/portable-computer-use-host.mjsindex ca645968..9e635f0b 100644--- a/evals/packages/labs/fixtures/portable-computer-use-host.mjs+++ b/evals/packages/labs/fixtures/portable-computer-use-host.mjs@@ -2,10 +2,13 @@// product integration and native preview without accounts or inference services.import { app, BrowserWindow, desktopCapturer } from "electron";import { createInterface } from "node:readline";+import { connect } from "node:net";+const channel = connect({ host: "127.0.0.1", port: Number(process.env.OMNIRUSH_FIXTURE_PORT) });+channel.on("connect", () => channel.write(JSON.stringify({ token: process.env.OMNIRUSH_FIXTURE_TOKEN }) + "\n"));import { checkComputerUsePermissions, getComputerUseMcpCommand, getComputerUseMcpEnvironment, getComputerUseState, computerUseAction } from "../../../../apps/desktop/electron/computer-use.mjs";app.setPath("userData", process.argv.at(-1));const ready = app.whenReady();-createInterface({ input: process.stdin }).on("line", async (line) => {+createInterface({ input: channel }).on("line", async (line) => {const request = JSON.parse(line);try {await ready; let result;@@ -21,7 +24,7 @@ createInterface({ input: process.stdin }).on("line", async (line) => {}else if (request.method === "action") { await computerUseAction(request.params); result = {}; }else throw new Error("Unknown host witness operation.");- process.stdout.write(JSON.stringify({ id: request.id, result }) + "\n");- } catch (error) { process.stdout.write(JSON.stringify({ id: request.id, error: { message: error.message } }) + "\n"); }+ channel.write(JSON.stringify({ id: request.id, result }) + "\n");+ } catch (error) { channel.write(JSON.stringify({ id: request.id, error: { message: error.message } }) + "\n"); }});-process.stdin.on("end", () => app.quit());+channel.on("close", () => app.quit());diff --git a/evals/specs/computer-use-portable-window.e2e.test.ts b/evals/specs/computer-use-portable-window.e2e.test.tsindex 49e78514..d506e60a 100644--- a/evals/specs/computer-use-portable-window.e2e.test.ts+++ b/evals/specs/computer-use-portable-window.e2e.test.ts@@ -74,6 +74,12 @@ test("Windows and Linux X11 Computer Use keeps native input within the approvedexpect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);await world.approve("resume");});+ await step("Desktop input from outside the controller automatically pauses control", async () => {+ await world.outsideInput();+ await expect.poll(async () => toolState(await world.call("computer_session_status", { session_id: session })).state).toBe("paused");+ expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);+ await world.approve("resume");+ });await step("Take over pauses control and only the person can continue", async () => {const observed = toolState(await world.call("computer_observe", { session_id: session }));await world.previewAction("pause");@@ -84,11 +90,21 @@ test("Windows and Linux X11 Computer Use keeps native input within the approvedexpect(toolState(await world.call("computer_session_status", { session_id: session })).state).toBe("active");expect(toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "invalidated-input", action: { type: "type", text: "Forbidden" } })).code).toBe("stale_observation");});- await step("Stop in the native preview revokes access and leaves both windows intact", async () => {+ const stoppedState = await step("Stop in the native preview interrupts typing and revokes access", async () => {+ const observed = toolState(await world.call("computer_observe", { session_id: session }));+ const pending = world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "interrupt-typing", action: { type: "type", text: "x".repeat(500) } });+ await expect.poll(async () => {+ const states = await world.state();+ return Array.isArray(states) && typeof states[0] === "object" && states[0] !== null && "draft" in states[0] && typeof states[0].draft === "string" && states[0].draft.includes("x");+ }).toBe(true);await world.previewAction("stop");+ expect(toolState(await pending)).toMatchObject({ ok: false, may_have_acted: true });expect(toolState(await world.call("computer_observe", { session_id: session })).code).toBe("session_unavailable");expect(await world.hostState()).toEqual([]);- expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);+ const states = await world.state();+ expect(states).toMatchObject([{ count: 1 }, { count: 0, draft: "Initial draft" }]);+ expect(JSON.stringify(states)).not.toContain("x".repeat(500));+ return states;});await step("Read-only approval never permits mouse or keyboard input", async () => {const pending = world.call("computer_open_session", { app_id: world.appId, pid: world.pid, mode: "observe", purpose: "Read the disposable fixture." });@@ -104,7 +120,7 @@ test("Windows and Linux X11 Computer Use keeps native input within the approvedexpect(await world.imagePixel(covered)).toMatchObject({ rgb: [238, 238, 238] });await world.closeClient();await expect.poll(() => world.hostState()).toEqual([]);- expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);+ expect(await world.state()).toEqual(stoppedState);});});diff --git a/evals/worlds/computer-use.ts b/evals/worlds/computer-use.tsindex 268d691e..b463b6f6 100644--- a/evals/worlds/computer-use.ts+++ b/evals/worlds/computer-use.ts@@ -4,6 +4,8 @@ import { tmpdir } from "node:os";import { dirname, join, resolve } from "node:path";import { fileURLToPath } from "node:url";import { createInterface } from "node:readline";+import { createServer } from "node:net";+import { randomBytes } from "node:crypto";import { createRequire } from "node:module";import { needs, SkipError } from "@omnirush/env";import type { Place, Seed } from "@omnirush/env";@@ -59,6 +61,64 @@ function pipeClient(executable: string, args: string[], environment: Record<stri};}+// Electron's Windows GUI executable does not provide a reliable stdio channel.+// Use an authenticated loopback socket for the disposable witness only.+async function nativeFixtureClient(executable: string, args: string[]) {+ const token = randomBytes(32).toString("hex");+ const server = createServer();+ await new Promise<void>((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); });+ const address = server.address();+ if (!address || typeof address === "string") throw new Error("No fixture endpoint.");+ const child = spawn(executable, args, { stdio: ["ignore", "ignore", "pipe"], env: { ...process.env, OMNIRUSH_FIXTURE_PORT: String(address.port), OMNIRUSH_FIXTURE_TOKEN: token } });+ let stderr = "";+ child.stderr.on("data", (chunk: Buffer) => { stderr = (stderr + chunk.toString()).slice(-2000); });+ const pending = new Map<number, { resolve(value: unknown): void; reject(error: Error): void; timer: NodeJS.Timeout }>();+ let nextId = 0;+ const socket = await new Promise<import("node:net").Socket>((resolve, reject) => {+ const timer = setTimeout(() => { server.close(); child.kill(); reject(new Error("Fixture connection timed out: " + stderr)); }, 15_000);+ child.once("error", (error) => { clearTimeout(timer); server.close(); reject(error); });+ child.once("exit", () => { clearTimeout(timer); server.close(); reject(new Error("Fixture exited: " + stderr)); });+ server.on("connection", (connected) => {+ const lines = createInterface({ input: connected });+ let authenticated = false;+ lines.on("line", (line) => {+ let message: unknown;+ try { message = JSON.parse(line); } catch { connected.destroy(); return; }+ if (!record(message)) { connected.destroy(); return; }+ if (!authenticated) {+ if (message.token !== token) { connected.destroy(); return; }+ authenticated = true; clearTimeout(timer); server.close(); resolve(connected); return;+ }+ if (typeof message.id !== "number") return;+ const request = pending.get(message.id); if (!request) return;+ pending.delete(message.id); clearTimeout(request.timer);+ if (message.error) request.reject(new Error(JSON.stringify(message.error))); else request.resolve(message.result);+ });+ connected.on("error", () => connected.destroy());+ connected.once("close", () => { lines.close(); for (const request of pending.values()) { clearTimeout(request.timer); request.reject(new Error("Fixture connection closed: " + stderr)); } pending.clear(); });+ });+ });+ return {+ pid: child.pid,+ request(method: string, params: Record<string, unknown> = {}, timeoutMs = 15_000) {+ const id = ++nextId;+ return new Promise<unknown>((resolve, reject) => {+ const timer = setTimeout(() => { pending.delete(id); reject(new Error("Fixture operation timed out: " + method + "; " + stderr)); }, timeoutMs);+ pending.set(id, { resolve, reject, timer }); socket.write(JSON.stringify({ id, method, params }) + "\n");+ });+ },+ async close() {+ socket.end();+ await new Promise<void>((resolve) => {+ if (child.exitCode !== null || child.signalCode !== null) { resolve(); return; }+ const timer = setTimeout(() => { child.kill(); resolve(); }, 2000);+ child.once("exit", () => { clearTimeout(timer); resolve(); });+ });+ socket.destroy();+ },+ };+}+export function toolState(value: unknown): Record<string, unknown> {if (!record(value) || !Array.isArray(value.content)) throw new Error("No tool content");const text = value.content.find((item: unknown) => record(item) && item.type === "text");@@ -256,8 +316,8 @@ export async function portableComputerUseWorld(_seed: Seed, { place }: { place:if (typeof executable !== "string") throw new Error("Electron executable unavailable.");const directory = await mkdtemp(join(tmpdir(), "omnirush-native-journey-"));const launchFlags = process.platform === "linux" ? ["--no-sandbox"] : [];- const fixture = pipeClient(executable, [...launchFlags, join(root, "evals/packages/labs/fixtures/portable-computer-use-app.mjs"), join(directory, "fixture")]);- const controller = pipeClient(executable, [...launchFlags, join(root, "evals/packages/labs/fixtures/portable-computer-use-host.mjs"), join(directory, "host")]);+ const fixture = await nativeFixtureClient(executable, [...launchFlags, join(root, "evals/packages/labs/fixtures/portable-computer-use-app.mjs"), join(directory, "fixture")]);+ const controller = await nativeFixtureClient(executable, [...launchFlags, join(root, "evals/packages/labs/fixtures/portable-computer-use-host.mjs"), join(directory, "host")]).catch(async (error) => { await fixture.close(); await rm(directory, { recursive: true, force: true }); throw error; });let client: ReturnType<typeof pipeClient> | undefined;let peer: ReturnType<typeof pipeClient> | undefined;try {@@ -299,6 +359,7 @@ export async function portableComputerUseWorld(_seed: Seed, { place }: { place:change: () => fixture.request("change"),focusOther: () => fixture.request("focus_other"),cover: () => fixture.request("cover"),+ outsideInput: () => fixture.request("outside_input"),imagePixel(reply: unknown) {if (!record(reply) || !Array.isArray(reply.content)) throw new Error("Missing image content.");const image = reply.content.find((item: unknown) => record(item) && item.type === "image");