Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

test: verify outside input and Stop using portable witnesses

Task Builder28292087577d
diff --git a/evals/packages/labs/fixtures/portable-computer-use-app.mjs b/evals/packages/labs/fixtures/portable-computer-use-app.mjs
index 96dc937b..041423a8 100644
--- a/evals/packages/labs/fixtures/portable-computer-use-app.mjs
+++ b/evals/packages/labs/fixtures/portable-computer-use-app.mjs
@@ -1,6 +1,11 @@
// Disposable native app used as an OS-level witness, never a real user's app.
import { app, BrowserWindow, screen, nativeImage } from "electron";
import { createInterface } from "node:readline";
+import { connect } from "node:net";
+const channel = connect({ host: "127.0.0.1", port: Number(process.env.OMNIRUSH_FIXTURE_PORT) });
+channel.on("connect", () => channel.write(JSON.stringify({ token: process.env.OMNIRUSH_FIXTURE_TOKEN }) + "\n"));
+import { createRequire } from "node:module";
+const requireDesktop = createRequire(new URL("../../../../apps/desktop/package.json", import.meta.url));
let windows = [];
app.setPath("userData", process.argv.at(-1));
const ready = app.whenReady().then(async () => {
@@ -10,13 +15,34 @@ const ready = app.whenReady().then(async () => {
await window.loadURL("data:text/html," + encodeURIComponent(html)); windows.push(window);
}
});
-createInterface({ input: process.stdin }).on("line", async (line) => {
+createInterface({ input: channel }).on("line", async (line) => {
const request = JSON.parse(line);
try {
await ready; let result;
if (request.method === "state") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript('({ count: window.counter, draft: document.getElementById("draft").value })')));
else if (request.method === "bounds") result = windows.map((w) => ({ bounds: w.getBounds(), contentBounds: w.getContentBounds(), scale: screen.getDisplayMatching(w.getBounds()).scaleFactor }));
else if (request.method === "focus_other") { windows[1].focus(); result = {}; }
+ else if (request.method === "outside_input") {
+ const koffi = requireDesktop("koffi");
+ if (process.platform === "win32") {
+ const keyboard = koffi.struct({ wVk: "uint16_t", wScan: "uint16_t", dwFlags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t" });
+ const mouse = koffi.struct({ dx: "int32_t", dy: "int32_t", mouseData: "uint32_t", dwFlags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t" });
+ const input = koffi.struct({ type: "uint32_t", value: koffi.union({ ki: keyboard, mi: mouse }) });
+ const send = koffi.load("user32.dll").func("uint32_t __stdcall SendInput(uint32_t count, void *input, int size)");
+ const data = Buffer.alloc(koffi.sizeof(input) * 2);
+ koffi.encode(data, input, [{ type: 1, value: { ki: { wVk: 16, wScan: 0, dwFlags: 0, time: 0, dwExtraInfo: 0 } } }, { type: 1, value: { ki: { wVk: 16, wScan: 0, dwFlags: 2, time: 0, dwExtraInfo: 0 } } }], 2);
+ if (send(2, data, koffi.sizeof(input)) !== 2) throw new Error("Outside keyboard stimulus unavailable.");
+ } else {
+ const x = koffi.load("libX11.so.6"), xt = koffi.load("libXtst.so.6"), display = x.func("void *XOpenDisplay(str name)")(null);
+ if (!display) throw new Error("Outside input display unavailable.");
+ try {
+ const code = x.func("uint8_t XKeysymToKeycode(void *display, ulong symbol)")(display, 0xffe1);
+ const send = xt.func("int XTestFakeKeyEvent(void *display, uint code, int down, ulong delay)");
+ send(display, code, 1, 0); send(display, code, 0, 0); x.func("int XSync(void *display, int discard)")(display, 0);
+ } finally { x.func("int XCloseDisplay(void *display)")(display); }
+ }
+ result = {};
+ }
else if (request.method === "cover") { windows[1].setBounds(windows[0].getBounds()); windows[1].setAlwaysOnTop(true); await windows[1].webContents.executeJavaScript('document.body.style.background="#ff00ff"'); windows[1].focus(); result = {}; }
else if (request.method === "image_pixel") { const image = nativeImage.createFromBuffer(Buffer.from(request.params.data, "base64")); const size = image.getSize(), pixels = image.toBitmap(); const offset = ((size.height - 50) * size.width + 20) * 4; result = { width: size.width, height: size.height, rgb: [pixels[offset + 2], pixels[offset + 1], pixels[offset]] }; }
else if (request.method === "change") { await windows[0].webContents.executeJavaScript('document.getElementById("draft").value="Changed by person"; document.body.style.background="#eeeeee"; new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))'); result = {}; }
@@ -24,7 +50,7 @@ createInterface({ input: process.stdin }).on("line", async (line) => {
else if (request.method === "restore") { windows[0].restore(); result = {}; }
else if (request.method === "close") { app.quit(); return; }
else throw new Error("Unknown fixture request.");
- process.stdout.write(JSON.stringify({ id: request.id, result }) + "\n");
- } catch (error) { process.stdout.write(JSON.stringify({ id: request.id, error: { message: error.message } }) + "\n"); }
+ channel.write(JSON.stringify({ id: request.id, result }) + "\n");
+ } catch (error) { channel.write(JSON.stringify({ id: request.id, error: { message: error.message } }) + "\n"); }
});
-process.stdin.on("end", () => app.quit());
+channel.on("close", () => app.quit());
diff --git a/evals/packages/labs/fixtures/portable-computer-use-host.mjs b/evals/packages/labs/fixtures/portable-computer-use-host.mjs
index ca645968..9e635f0b 100644
--- a/evals/packages/labs/fixtures/portable-computer-use-host.mjs
+++ b/evals/packages/labs/fixtures/portable-computer-use-host.mjs
@@ -2,10 +2,13 @@
// product integration and native preview without accounts or inference services.
import { app, BrowserWindow, desktopCapturer } from "electron";
import { createInterface } from "node:readline";
+import { connect } from "node:net";
+const channel = connect({ host: "127.0.0.1", port: Number(process.env.OMNIRUSH_FIXTURE_PORT) });
+channel.on("connect", () => channel.write(JSON.stringify({ token: process.env.OMNIRUSH_FIXTURE_TOKEN }) + "\n"));
import { checkComputerUsePermissions, getComputerUseMcpCommand, getComputerUseMcpEnvironment, getComputerUseState, computerUseAction } from "../../../../apps/desktop/electron/computer-use.mjs";
app.setPath("userData", process.argv.at(-1));
const ready = app.whenReady();
-createInterface({ input: process.stdin }).on("line", async (line) => {
+createInterface({ input: channel }).on("line", async (line) => {
const request = JSON.parse(line);
try {
await ready; let result;
@@ -21,7 +24,7 @@ createInterface({ input: process.stdin }).on("line", async (line) => {
}
else if (request.method === "action") { await computerUseAction(request.params); result = {}; }
else throw new Error("Unknown host witness operation.");
- process.stdout.write(JSON.stringify({ id: request.id, result }) + "\n");
- } catch (error) { process.stdout.write(JSON.stringify({ id: request.id, error: { message: error.message } }) + "\n"); }
+ channel.write(JSON.stringify({ id: request.id, result }) + "\n");
+ } catch (error) { channel.write(JSON.stringify({ id: request.id, error: { message: error.message } }) + "\n"); }
});
-process.stdin.on("end", () => app.quit());
+channel.on("close", () => app.quit());
diff --git a/evals/specs/computer-use-portable-window.e2e.test.ts b/evals/specs/computer-use-portable-window.e2e.test.ts
index 49e78514..d506e60a 100644
--- a/evals/specs/computer-use-portable-window.e2e.test.ts
+++ b/evals/specs/computer-use-portable-window.e2e.test.ts
@@ -74,6 +74,12 @@ test("Windows and Linux X11 Computer Use keeps native input within the approved
expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);
await world.approve("resume");
});
+ await step("Desktop input from outside the controller automatically pauses control", async () => {
+ await world.outsideInput();
+ await expect.poll(async () => toolState(await world.call("computer_session_status", { session_id: session })).state).toBe("paused");
+ expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);
+ await world.approve("resume");
+ });
await step("Take over pauses control and only the person can continue", async () => {
const observed = toolState(await world.call("computer_observe", { session_id: session }));
await world.previewAction("pause");
@@ -84,11 +90,21 @@ test("Windows and Linux X11 Computer Use keeps native input within the approved
expect(toolState(await world.call("computer_session_status", { session_id: session })).state).toBe("active");
expect(toolState(await world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "invalidated-input", action: { type: "type", text: "Forbidden" } })).code).toBe("stale_observation");
});
- await step("Stop in the native preview revokes access and leaves both windows intact", async () => {
+ const stoppedState = await step("Stop in the native preview interrupts typing and revokes access", async () => {
+ const observed = toolState(await world.call("computer_observe", { session_id: session }));
+ const pending = world.call("computer_act", { session_id: session, observation_id: observed.observation_id, request_id: "interrupt-typing", action: { type: "type", text: "x".repeat(500) } });
+ await expect.poll(async () => {
+ const states = await world.state();
+ return Array.isArray(states) && typeof states[0] === "object" && states[0] !== null && "draft" in states[0] && typeof states[0].draft === "string" && states[0].draft.includes("x");
+ }).toBe(true);
await world.previewAction("stop");
+ expect(toolState(await pending)).toMatchObject({ ok: false, may_have_acted: true });
expect(toolState(await world.call("computer_observe", { session_id: session })).code).toBe("session_unavailable");
expect(await world.hostState()).toEqual([]);
- expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);
+ const states = await world.state();
+ expect(states).toMatchObject([{ count: 1 }, { count: 0, draft: "Initial draft" }]);
+ expect(JSON.stringify(states)).not.toContain("x".repeat(500));
+ return states;
});
await step("Read-only approval never permits mouse or keyboard input", async () => {
const pending = world.call("computer_open_session", { app_id: world.appId, pid: world.pid, mode: "observe", purpose: "Read the disposable fixture." });
@@ -104,7 +120,7 @@ test("Windows and Linux X11 Computer Use keeps native input within the approved
expect(await world.imagePixel(covered)).toMatchObject({ rgb: [238, 238, 238] });
await world.closeClient();
await expect.poll(() => world.hostState()).toEqual([]);
- expect(await world.state()).toEqual([{ count: 1, draft: "Changed by person" }, { count: 0, draft: "Initial draft" }]);
+ expect(await world.state()).toEqual(stoppedState);
});
});
diff --git a/evals/worlds/computer-use.ts b/evals/worlds/computer-use.ts
index 268d691e..b463b6f6 100644
--- a/evals/worlds/computer-use.ts
+++ b/evals/worlds/computer-use.ts
@@ -4,6 +4,8 @@ import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { createInterface } from "node:readline";
+import { createServer } from "node:net";
+import { randomBytes } from "node:crypto";
import { createRequire } from "node:module";
import { needs, SkipError } from "@omnirush/env";
import type { Place, Seed } from "@omnirush/env";
@@ -59,6 +61,64 @@ function pipeClient(executable: string, args: string[], environment: Record<stri
};
}
+// Electron's Windows GUI executable does not provide a reliable stdio channel.
+// Use an authenticated loopback socket for the disposable witness only.
+async function nativeFixtureClient(executable: string, args: string[]) {
+ const token = randomBytes(32).toString("hex");
+ const server = createServer();
+ await new Promise<void>((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); });
+ const address = server.address();
+ if (!address || typeof address === "string") throw new Error("No fixture endpoint.");
+ const child = spawn(executable, args, { stdio: ["ignore", "ignore", "pipe"], env: { ...process.env, OMNIRUSH_FIXTURE_PORT: String(address.port), OMNIRUSH_FIXTURE_TOKEN: token } });
+ let stderr = "";
+ child.stderr.on("data", (chunk: Buffer) => { stderr = (stderr + chunk.toString()).slice(-2000); });
+ const pending = new Map<number, { resolve(value: unknown): void; reject(error: Error): void; timer: NodeJS.Timeout }>();
+ let nextId = 0;
+ const socket = await new Promise<import("node:net").Socket>((resolve, reject) => {
+ const timer = setTimeout(() => { server.close(); child.kill(); reject(new Error("Fixture connection timed out: " + stderr)); }, 15_000);
+ child.once("error", (error) => { clearTimeout(timer); server.close(); reject(error); });
+ child.once("exit", () => { clearTimeout(timer); server.close(); reject(new Error("Fixture exited: " + stderr)); });
+ server.on("connection", (connected) => {
+ const lines = createInterface({ input: connected });
+ let authenticated = false;
+ lines.on("line", (line) => {
+ let message: unknown;
+ try { message = JSON.parse(line); } catch { connected.destroy(); return; }
+ if (!record(message)) { connected.destroy(); return; }
+ if (!authenticated) {
+ if (message.token !== token) { connected.destroy(); return; }
+ authenticated = true; clearTimeout(timer); server.close(); resolve(connected); return;
+ }
+ if (typeof message.id !== "number") return;
+ const request = pending.get(message.id); if (!request) return;
+ pending.delete(message.id); clearTimeout(request.timer);
+ if (message.error) request.reject(new Error(JSON.stringify(message.error))); else request.resolve(message.result);
+ });
+ connected.on("error", () => connected.destroy());
+ connected.once("close", () => { lines.close(); for (const request of pending.values()) { clearTimeout(request.timer); request.reject(new Error("Fixture connection closed: " + stderr)); } pending.clear(); });
+ });
+ });
+ return {
+ pid: child.pid,
+ request(method: string, params: Record<string, unknown> = {}, timeoutMs = 15_000) {
+ const id = ++nextId;
+ return new Promise<unknown>((resolve, reject) => {
+ const timer = setTimeout(() => { pending.delete(id); reject(new Error("Fixture operation timed out: " + method + "; " + stderr)); }, timeoutMs);
+ pending.set(id, { resolve, reject, timer }); socket.write(JSON.stringify({ id, method, params }) + "\n");
+ });
+ },
+ async close() {
+ socket.end();
+ await new Promise<void>((resolve) => {
+ if (child.exitCode !== null || child.signalCode !== null) { resolve(); return; }
+ const timer = setTimeout(() => { child.kill(); resolve(); }, 2000);
+ child.once("exit", () => { clearTimeout(timer); resolve(); });
+ });
+ socket.destroy();
+ },
+ };
+}
+
export function toolState(value: unknown): Record<string, unknown> {
if (!record(value) || !Array.isArray(value.content)) throw new Error("No tool content");
const text = value.content.find((item: unknown) => record(item) && item.type === "text");
@@ -256,8 +316,8 @@ export async function portableComputerUseWorld(_seed: Seed, { place }: { place:
if (typeof executable !== "string") throw new Error("Electron executable unavailable.");
const directory = await mkdtemp(join(tmpdir(), "omnirush-native-journey-"));
const launchFlags = process.platform === "linux" ? ["--no-sandbox"] : [];
- const fixture = pipeClient(executable, [...launchFlags, join(root, "evals/packages/labs/fixtures/portable-computer-use-app.mjs"), join(directory, "fixture")]);
- const controller = pipeClient(executable, [...launchFlags, join(root, "evals/packages/labs/fixtures/portable-computer-use-host.mjs"), join(directory, "host")]);
+ const fixture = await nativeFixtureClient(executable, [...launchFlags, join(root, "evals/packages/labs/fixtures/portable-computer-use-app.mjs"), join(directory, "fixture")]);
+ const controller = await nativeFixtureClient(executable, [...launchFlags, join(root, "evals/packages/labs/fixtures/portable-computer-use-host.mjs"), join(directory, "host")]).catch(async (error) => { await fixture.close(); await rm(directory, { recursive: true, force: true }); throw error; });
let client: ReturnType<typeof pipeClient> | undefined;
let peer: ReturnType<typeof pipeClient> | undefined;
try {
@@ -299,6 +359,7 @@ export async function portableComputerUseWorld(_seed: Seed, { place }: { place:
change: () => fixture.request("change"),
focusOther: () => fixture.request("focus_other"),
cover: () => fixture.request("cover"),
+ outsideInput: () => fixture.request("outside_input"),
imagePixel(reply: unknown) {
if (!record(reply) || !Array.isArray(reply.content)) throw new Error("Missing image content.");
const image = reply.content.find((item: unknown) => record(item) && item.type === "image");