Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: computer use review fixes

ogsamrat6135cf9d1efe
diff --git a/apps/desktop/electron/computer-use-host.mjs b/apps/desktop/electron/computer-use-host.mjs
index 6e91b1cf..c9e0a1ee 100644
--- a/apps/desktop/electron/computer-use-host.mjs
+++ b/apps/desktop/electron/computer-use-host.mjs
@@ -147,7 +147,9 @@ export function createMacNativeWorker({ executable, args = ["worker"], captureLo
// Painted here on every capture (the host paints the same boxes again, which changes nothing), so
// the host's stale check compares a masked observation with a masked fresh capture.
paintBoxes?.(frame, secure);
- return { width: frame.width, height: frame.height, data: frame.data, secure, caret: frame.caret ?? null, ocr: frame.ocr === "client" ? "client" : "none", credential_boxes: text.length, source: frame.source };
+ // The recognized lines (boxes + text) for the host's per-act rules (risky targets); never sent to the model.
+ const texts = Array.isArray(frame.text_boxes) ? frame.text_boxes.filter((l) => record(l) && typeof l.text === "string" && record(l.box)).map((l) => ({ x: l.box.x, y: l.box.y, width: l.box.width, height: l.box.height, text: l.text })) : null;
+ return { width: frame.width, height: frame.height, data: frame.data, secure, secure_fields: Array.isArray(frame.secure) ? frame.secure : [], ...(frame.ocr === "client" && texts ? { texts } : {}), caret: frame.caret ?? null, ocr: frame.ocr === "client" ? "client" : "none", credential_boxes: text.length, source: frame.source };
},
kill() { child.kill(); },
async close() { if (closed) return; await raw("shutdown", {}, 3000).catch(() => {}); closed = true; if (child.exitCode === null) child.kill(); },
diff --git a/apps/desktop/electron/computer-use.mjs b/apps/desktop/electron/computer-use.mjs
index 4e46f341..34e13fb0 100644
--- a/apps/desktop/electron/computer-use.mjs
+++ b/apps/desktop/electron/computer-use.mjs
@@ -12,6 +12,7 @@
// chat's permission mode, are answered in the main window (desktop IPC from
// its main frame, input checked for synthetic events), never by the agent.
import { spawn } from "node:child_process";
+import { randomBytes } from "node:crypto";
import { existsSync } from "node:fs";
import { createRequire } from "node:module";
import path from "node:path";
@@ -24,7 +25,22 @@ import { createComputerPreview } from "./computer-use-preview.mjs";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const require = createRequire(import.meta.url);
const STATUS_CHANNEL = "omnirush:computer-use:status";
+// A per-launch secret, held in this process only (the local server runs in this process and reads it here):
+// the host and event routes take it, so a client token read from disk cannot replace the host or forge events.
+const HOST_SECRET = randomBytes(32).toString("hex");
+globalThis[Symbol.for("omnirush.computer-use.host-secret")] = HOST_SECRET;
+const secretHeaders = { "x-omnirush-computer-use-secret": HOST_SECRET };
+// The profile folder the local server expects the host in (computer-use/desktop.ts expectedHostCommand).
+const profileDir = () => process.env.OMNIRUSH_DESKTOP_USER_DATA_DIR?.trim() || app.getPath("userData");
const SYNTHETIC_MESSAGE = "Use your own mouse or keyboard to approve.";
+// Answers need a physical press in this app's window. OMNIRUSH_COMPUTER_USE_TEST_ANSWERS=1 (rigs only: the
+// app's own environment, which the agent cannot set for a running app) accepts remote-debugging clicks.
+const testAnswers = process.env.OMNIRUSH_COMPUTER_USE_TEST_ANSWERS === "1";
+async function personAnswered() {
+ if (!host) return false;
+ if (await host.syntheticInput()) return false;
+ return testAnswers || host.physicalInput(5000, { foregroundPid: process.pid });
+}
let deps = null;
let server = null; // the last /computer-use/state answer
@@ -148,7 +164,7 @@ async function hostContext() {
async function reportEvent(event) {
const session = host?.state().find((s) => s.id === event.session_id);
- await serverFetch("/computer-use/event", { method: "POST", body: JSON.stringify({ event, ...(session?.chat ? { chat: session.chat } : {}) }) }).catch(() => {});
+ await serverFetch("/computer-use/event", { method: "POST", headers: secretHeaders, body: JSON.stringify({ event, ...(session?.chat ? { chat: session.chat } : {}) }) }).catch(() => {});
}
async function startHost() {
@@ -174,11 +190,11 @@ async function startHost() {
preview.bind(created);
}
host = created;
- listener = await created.listen(path.join(app.getPath("userData"), "computer-use"));
+ listener = await created.listen(path.join(profileDir(), "computer-use"));
const relay = app.isPackaged ? path.join(process.resourcesPath, "computer-use", "relay.mjs") : path.join(__dirname, "computer-use-relay.mjs");
const command = [process.execPath, relay, listener.discoveryPath];
const environment = { ELECTRON_RUN_AS_NODE: "1" };
- await serverFetch("/computer-use/host", { method: "POST", body: JSON.stringify({ command, environment }) });
+ await serverFetch("/computer-use/host", { method: "POST", headers: secretHeaders, body: JSON.stringify({ command, environment }) });
pushStatus();
return created;
})().catch((error) => { hostPromise = null; host = null; throw error; });
@@ -187,7 +203,7 @@ async function startHost() {
async function stopHost(by = "server") {
const current = host;
- await serverFetch("/computer-use/host", { method: "POST", body: JSON.stringify({ host: null }) }).catch(() => {});
+ await serverFetch("/computer-use/host", { method: "POST", headers: secretHeaders, body: JSON.stringify({ host: null }) }).catch(() => {});
if (!current) return;
await current.stopAll(by).catch(() => {});
for (const entry of pending.values()) entry.resolve({ decision: "deny" });
@@ -241,14 +257,14 @@ async function computerUseAnswer(answer) {
const entry = pending.get(answer?.request_id);
if (!entry) throw new Error("This request has ended.");
const allowing = answer.decision !== "deny";
- if (allowing && host && await host.syntheticInput()) throw new Error(SYNTHETIC_MESSAGE);
+ if (allowing && !(await personAnswered())) throw new Error(SYNTHETIC_MESSAGE);
const decision = entry.request.kind === "open" ? (["chat", "once", "deny"].includes(answer.decision) ? answer.decision : "deny") : (answer.decision === "allow" ? "allow" : "deny");
entry.resolve({ decision, ...(typeof answer.window_id === "string" ? { window_id: answer.window_id } : {}) });
}
async function computerUseControl(value) {
if (!host) throw new Error("Computer use is not running.");
- if (value?.action === "continue" && await host.syntheticInput()) throw new Error(SYNTHETIC_MESSAGE);
+ if (value?.action === "continue" && !(await personAnswered())) throw new Error(SYNTHETIC_MESSAGE);
await host.control({ session_id: value?.session_id, action: value?.action });
pushStatus();
}
diff --git a/apps/server/src/computer-use/contract-policy.test.ts b/apps/server/src/computer-use/contract-policy.test.ts
index 7d0a83a7..6d7f8182 100644
--- a/apps/server/src/computer-use/contract-policy.test.ts
+++ b/apps/server/src/computer-use/contract-policy.test.ts
@@ -8,6 +8,8 @@ import {
INSTRUCTIONS,
isComputerTool,
isFailure,
+ keyFromKeysymName,
+ keyFromVirtualKey,
mapPoint,
mcpResult,
modelSize,
@@ -62,6 +64,13 @@ describe("contract v2: tools", () => {
});
});
+describe("contract v2: pressed keys as a terminal sees them", () => {
+ test("X11 keysym names and Windows virtual keys", () => {
+ expect(["a", "Y", "1", "Return", "Escape", "space", "F5", "Shift_L", "BackSpace"].map(keyFromKeysymName)).toEqual(["a", "y", "1", "Enter", "Escape", "Space", "F5", null, "Backspace"]);
+ expect([0x41, 0x5a, 0x30, 0x0d, 0x1b, 0x74, 0x10, 0x63].map(keyFromVirtualKey)).toEqual(["a", "z", "0", "Enter", "Escape", "F5", null, "3"]);
+ });
+});
+
describe("contract v2: validation", () => {
test("act needs intent, ids and an action", () => {
const base = { session_id: "s1", observation_id: "o1", request_id: "r1", intent: "Open the File menu", action: { type: "click", x: 1, y: 1 } };
@@ -241,6 +250,15 @@ describe("policy: approvals per permission mode", () => {
expect(documentOf("● main.ts - Code", "Code")).toBe("main.ts");
});
+ test("Normal, no readable text at the target: a click or Enter right after typing asks", () => {
+ expect(actApproval({ ...plain, target: {}, typed_before: true }).reasons).toEqual([{ code: "submit_after_typing" }]);
+ expect(actApproval({ ...plain, action: { type: "key", key: "Enter" }, target: {}, typed_before: true }).ask).toBe(true);
+ expect(actApproval({ ...plain, target: {}, typed_before: false }).ask).toBe(false);
+ // With text the host read there, the target rules decide instead.
+ expect(actApproval({ ...plain, target: { text: "Bold" }, typed_before: true }).ask).toBe(false);
+ expect(actApproval({ ...plain, mode: "auto", target: {}, typed_before: true }).ask).toBe(false);
+ });
+
test("shell screen capture asks; sub-agents get no computer tools", () => {
expect(SCREEN_BASH_RULES["*xdotool*"]).toBe("ask");
expect(SCREEN_BASH_RULES["*screencapture*"]).toBe("ask");
diff --git a/apps/server/src/computer-use/desktop.test.ts b/apps/server/src/computer-use/desktop.test.ts
index 17cddb71..a5933e0a 100644
--- a/apps/server/src/computer-use/desktop.test.ts
+++ b/apps/server/src/computer-use/desktop.test.ts
@@ -7,6 +7,9 @@ import path from "node:path";
import omnirushMcpPolicy from "../opencode-plugins/omnirush-mcp-policy.js";
import { MediaStore } from "../session-archive/media-store.js";
+import type { Route } from "../routes/registry.js";
+import { registerComputerUseRoutes } from "./routes.js";
+import { expectedHostCommand, HOST_SECRET_KEY } from "./desktop.js";
import { computerUseBuiltins, computerUseDir, computerUseSettingFor, createChatTracker, permissionModeName, rememberComputerUseSwitch, sessionEventOf, writeHostFile } from "./desktop.js";
import { createMediaRewriter, usesComputer } from "./media-capture.js";
import { encodePng } from "./shared/image.mjs";
@@ -37,10 +40,16 @@ describe("switch and built-in entry", () => {
expect(computerUseSettingFor(dir, { OMNIRUSH_COMPUTER_USE: "0" }).enabled).toBe(false);
expect(computerUseBuiltins(dir, {})).toEqual({ "computer-use": null });
expect(writeHostFile(dir, { command: [process.execPath, "/x/relay.mjs", "/x/connection.json"], environment: { ELECTRON_RUN_AS_NODE: "1" } })).toBe(true);
- expect(computerUseBuiltins(dir, {})["computer-use"]).toMatchObject({ type: "local", command: [process.execPath, "/x/relay.mjs", "/x/connection.json"], environment: { ELECTRON_RUN_AS_NODE: "1" }, enabled: true, timeout: 300_000 });
+ const pinned = { OMNIRUSH_COMPUTER_USE_COMMAND: JSON.stringify({ command: [process.execPath, "/x/relay.mjs", "/x/connection.json"], environment: { ELECTRON_RUN_AS_NODE: "1" } }) };
+ // The command comes only from the app's environment; host.json must agree with it.
+ expect(computerUseBuiltins(dir, {})).toEqual({ "computer-use": null });
+ expect(computerUseBuiltins(dir, pinned)["computer-use"]).toMatchObject({ type: "local", command: [process.execPath, "/x/relay.mjs", "/x/connection.json"], environment: { ELECTRON_RUN_AS_NODE: "1" }, enabled: true, timeout: 300_000 });
+ writeHostFile(dir, { command: [process.execPath, "/tmp/evil.mjs"], environment: {} });
+ expect(computerUseBuiltins(dir, pinned)).toEqual({ "computer-use": null });
+ writeHostFile(dir, { command: [process.execPath, "/x/relay.mjs", "/x/connection.json"], environment: { ELECTRON_RUN_AS_NODE: "1" } });
expect(rememberComputerUseSwitch(state, {})).toBe(true);
expect(computerUseBuiltins(dir, {})).toEqual({ "computer-use": null });
- expect(computerUseBuiltins(dir, { OMNIRUSH_COMPUTER_USE: "1" })["computer-use"]).not.toBeNull();
+ expect(computerUseBuiltins(dir, { ...pinned, OMNIRUSH_COMPUTER_USE: "1" })["computer-use"]).not.toBeNull();
expect(writeHostFile(dir, null)).toBe(true);
expect(computerUseBuiltins(undefined, {})).toEqual({ "computer-use": null });
});
@@ -75,8 +84,9 @@ describe("engine MCP plugin", () => {
const dir = computerUseDir(state);
const reports = path.join(state, "reports");
writeHostFile(dir, { command: [process.execPath, "/x/relay.mjs", "/x/c.json"], environment: {} });
- const { config } = await run({ OMNIRUSH_COMPUTER_USE_DIR: dir, OMNIRUSH_COMPUTER_USE: "1", OMNIRUSH_MCP_REPORT_DIR: reports }, {
- mcp: { "computer-use": { type: "local", command: ["npx", "-y", "evil"] } },
+ const pinnedEnv = { OMNIRUSH_COMPUTER_USE_COMMAND: JSON.stringify({ command: [process.execPath, "/x/relay.mjs", "/x/c.json"], environment: {} }) };
+ const { config } = await run({ ...pinnedEnv, OMNIRUSH_COMPUTER_USE_DIR: dir, OMNIRUSH_COMPUTER_USE: "1", OMNIRUSH_MCP_REPORT_DIR: reports }, {
+ mcp: { "computer-use": { type: "local", command: ["npx", "-y", "evil"] }, "computer-use_computer": { type: "local", command: ["npx", "-y", "evil2"] } },
permission: { "*": "ask", bash: { "*": "allow", "rm -r*": "ask" } },
agent: { custom: { mode: "subagent" } },
});
@@ -89,6 +99,7 @@ describe("engine MCP plugin", () => {
for (const name of ["general", "explore", "custom"]) expect(agents[name]?.permission["computer-use_*"]).toBe("deny");
const report = JSON.parse(readFileSync(path.join(reports, readdirName(reports)), "utf8")) as { decisions: Array<{ name: string; code: string }> };
expect(report.decisions.find((d) => d.name === "computer-use")).toMatchObject({ code: "reserved" });
+ expect(report.decisions.find((d) => d.name === "computer-use_computer")).toMatchObject({ code: "reserved" });
});
test("off: nothing is added and the config is as before", async () => {
@@ -101,7 +112,8 @@ describe("engine MCP plugin", () => {
const state = tmp();
const dir = computerUseDir(state);
writeHostFile(dir, { command: [process.execPath, "/x/relay.mjs", "/x/c.json"], environment: {} });
- const { config } = await run({ OMNIRUSH_COMPUTER_USE_DIR: dir, OMNIRUSH_COMPUTER_USE: "1" }, { permission: { bash: { "*": "allow" } } });
+ const pinnedEnv = { OMNIRUSH_COMPUTER_USE_COMMAND: JSON.stringify({ command: [process.execPath, "/x/relay.mjs", "/x/c.json"], environment: {} }) };
+ const { config } = await run({ ...pinnedEnv, OMNIRUSH_COMPUTER_USE_DIR: dir, OMNIRUSH_COMPUTER_USE: "1" }, { permission: { bash: { "*": "allow" } } });
expect((config.permission as { bash: Record<string, string> }).bash).toEqual({ "*": "allow" });
});
@@ -118,6 +130,42 @@ describe("engine MCP plugin", () => {
});
});
+describe("sub-agents and the routes only Electron main may call", () => {
+ test("a computer tool call from a sub-agent session is refused", async () => {
+ const client = { session: { get: async ({ path: p }: { path: { id: string } }) => ({ data: p.id === "ses_child" ? { parentID: "ses_main" } : {} }) } };
+ const plugin = await omnirushMcpPolicy({ directory: tmp(), client });
+ await expect(plugin["tool.execute.before"]({ tool: "computer-use_computer_act", sessionID: "ses_child", args: {} }, { args: {} })).rejects.toThrow(/sub-agents/);
+ await plugin["tool.execute.before"]({ tool: "computer-use_computer_act", sessionID: "ses_main", args: {} }, { args: {} });
+ });
+
+ test("host and event routes need the per-launch secret and accept only the app's own command", async () => {
+ const routes: Route[] = [];
+ const state = tmp();
+ const recorded: unknown[] = [];
+ registerComputerUseRoutes({ routes, stateDir: () => state, jsonResponse: (d, status = 200) => new Response(JSON.stringify(d), { status }), readJsonBody: async (r) => r.json(), approvalMode: async () => "guarded", readCapabilities: async () => ({}), applyEngine: async () => undefined, chats: createChatTracker(), runningSessions: async () => null, recordTrace: (...args) => recorded.push(args) });
+ const hit = (method: string, p: string, body: unknown, headers: Record<string, string> = {}) => {
+ const route = routes.find((r) => r.method === method && r.regex.test(p));
+ if (!route) throw new Error("no route");
+ return route.handler({ request: new Request(`http://x${p}`, { method, body: JSON.stringify(body), headers }), url: new URL(`http://x${p}`), params: {} } as never);
+ };
+ const saved = process.env.OMNIRUSH_DESKTOP_USER_DATA_DIR;
+ process.env.OMNIRUSH_DESKTOP_USER_DATA_DIR = "/profile";
+ (globalThis as Record<symbol, unknown>)[HOST_SECRET_KEY] = "s".repeat(64);
+ try {
+ const expected = expectedHostCommand();
+ expect((await hit("POST", "/computer-use/host", expected)).status).toBe(403);
+ expect((await hit("POST", "/computer-use/host", { command: [process.execPath, "/tmp/evil.mjs"], environment: {} }, { "x-omnirush-computer-use-secret": "s".repeat(64) })).status).toBe(400);
+ expect((await hit("POST", "/computer-use/host", expected, { "x-omnirush-computer-use-secret": "s".repeat(64) })).status).toBe(200);
+ expect((await hit("POST", "/computer-use/event", { event: { event: "opened", by: "agent" }, chat: "ses_abc" })).status).toBe(403);
+ expect((await hit("POST", "/computer-use/event", { event: { event: "opened", by: "agent" }, chat: "ses_abc" }, { "x-omnirush-computer-use-secret": "s".repeat(64) })).status).toBe(200);
+ expect(recorded.length).toBe(1);
+ } finally {
+ delete (globalThis as Record<symbol, unknown>)[HOST_SECRET_KEY];
+ if (saved === undefined) delete process.env.OMNIRUSH_DESKTOP_USER_DATA_DIR; else process.env.OMNIRUSH_DESKTOP_USER_DATA_DIR = saved;
+ }
+ });
+});
+
function readdirName(dir: string): string {
const name = readdirSync(dir).find((n) => n.endsWith(".json"));
if (!name) throw new Error("no report");
diff --git a/apps/server/src/computer-use/desktop.ts b/apps/server/src/computer-use/desktop.ts
index e57bd8c6..5bb35cc5 100644
--- a/apps/server/src/computer-use/desktop.ts
+++ b/apps/server/src/computer-use/desktop.ts
@@ -14,8 +14,10 @@
* On, in this order: OMNIRUSH_COMPUTER_USE=1/0 in the environment, then the
* server's block, else off (policy.mjs computerUseSetting).
*/
+import { timingSafeEqual } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
+import { fileURLToPath } from "node:url";
import { SERVER_ID } from "./shared/contract.mjs";
import { computerUseSetting, type ComputerUseSetting } from "./shared/policy.mjs";
@@ -56,8 +58,39 @@ export function mediaIndexDir(stateDir: string): string {
}
/** The engine's environment for its plugins. */
-export function computerUseEngineEnv(stateDir: string): Record<string, string> {
- return { [COMPUTER_USE_DIR_ENV]: computerUseDir(stateDir) };
+export const COMPUTER_USE_COMMAND_ENV = "OMNIRUSH_COMPUTER_USE_COMMAND";
+
+/**
+ * The only command the built-in server may run: this app's own binary, the
+ * bundled relay and the host's discovery file in the app's profile. Fixed by
+ * the app (the engine gets it in its environment); never read from a file or
+ * a request. Null outside the desktop app (no profile folder known).
+ */
+export function expectedHostCommand(env: Record<string, string | undefined> = process.env): ComputerUseHostFile | null {
+ const profile = env.OMNIRUSH_DESKTOP_USER_DATA_DIR?.trim();
+ if (!profile) return null;
+ const resources = (process as { resourcesPath?: string }).resourcesPath;
+ const packaged = resources ? join(resources, "computer-use", "relay.mjs") : null;
+ const relay = packaged && existsSync(packaged) ? packaged : fileURLToPath(new URL("../../../desktop/electron/computer-use-relay.mjs", import.meta.url));
+ return { command: [process.execPath, relay, join(profile, "computer-use", "connection.json")], environment: { ELECTRON_RUN_AS_NODE: "1" } };
+}
+
+export function computerUseEngineEnv(stateDir: string, env: Record<string, string | undefined> = process.env): Record<string, string> {
+ const command = expectedHostCommand(env);
+ return { [COMPUTER_USE_DIR_ENV]: computerUseDir(stateDir), ...(command ? { [COMPUTER_USE_COMMAND_ENV]: JSON.stringify(command) } : {}) };
+}
+
+/** The per-launch secret Electron main holds (in this process only: never on disk, never in an environment). */
+export const HOST_SECRET_KEY = Symbol.for("omnirush.computer-use.host-secret");
+export function hostSecretMatches(given: string | null | undefined): boolean {
+ const held: unknown = (globalThis as Record<symbol, unknown>)[HOST_SECRET_KEY];
+ if (typeof held !== "string" || held.length < 32 || typeof given !== "string") return false;
+ const a = Buffer.from(held), b = Buffer.from(given);
+ return a.length === b.length && timingSafeEqual(a, b);
+}
+
+export function sameHostCommand(a: ComputerUseHostFile | null, b: ComputerUseHostFile | null): boolean {
+ return Boolean(a && b) && JSON.stringify(a?.command) === JSON.stringify(b?.command) && JSON.stringify(a?.environment) === JSON.stringify(b?.environment);
}
/** The block as last seen (null: never seen). */
@@ -113,9 +146,12 @@ export function writeHostFile(dir: string, host: ComputerUseHostFile | null): bo
export function computerUseBuiltins(dir: string | null | undefined, env: Record<string, string | undefined> = process.env): Record<string, Record<string, unknown> | null> {
if (!dir) return { [SERVER_ID]: null };
const setting = computerUseSettingFor(dir, env);
+ // The command comes from the app (the engine's environment); host.json only says the host runs, and must agree.
+ let expected: ComputerUseHostFile | null = null;
+ try { expected = JSON.parse(env[COMPUTER_USE_COMMAND_ENV] ?? "null") as ComputerUseHostFile | null; } catch { expected = null; }
const host = setting.enabled && setting.real_screen ? readHostFile(dir) : null;
- if (!host || !existsSync(host.command[0] ?? "")) return { [SERVER_ID]: null };
- return { [SERVER_ID]: { type: "local", command: host.command, environment: host.environment, enabled: true, timeout: COMPUTER_USE_MCP_TIMEOUT_MS, title: "Computer use" } };
+ if (!host || !expected || !sameHostCommand(host, expected) || !existsSync(expected.command[0] ?? "")) return { [SERVER_ID]: null };
+ return { [SERVER_ID]: { type: "local", command: expected.command, environment: expected.environment, enabled: true, timeout: COMPUTER_USE_MCP_TIMEOUT_MS, title: "Computer use" } };
}
/** The desktop's approval mode as the shared rules name it. */
diff --git a/apps/server/src/computer-use/media-image.test.ts b/apps/server/src/computer-use/media-image.test.ts
index 25748265..56a04503 100644
--- a/apps/server/src/computer-use/media-image.test.ts
+++ b/apps/server/src/computer-use/media-image.test.ts
@@ -229,6 +229,19 @@ describe("media references", () => {
expect(rewriteMessageMedia(untouched, { spool, index: createMediaIndex(path.join(dir, "j.json")), limits: {} }).message).toBe(untouched);
});
+ test("only computer use parts by default: a playwright screenshot stays exactly as it was", () => {
+ const dir = tmp();
+ const spool = createMediaSpool(path.join(dir, "spool"));
+ const png = encodePng(frame(30, 20, 31));
+ const part = { ...toolPart("pw", [dataUrl("image/png", png)]), tool: "playwright_browser_take_screenshot" };
+ const message = { parts: [part] };
+ const out = rewriteMessageMedia(message, { spool, index: createMediaIndex(path.join(dir, "i.json")), limits: {} });
+ expect(out.message).toBe(message);
+ expect(out.files).toEqual([]);
+ const all = rewriteMessageMedia(message, { spool, index: createMediaIndex(path.join(dir, "j.json")), limits: {}, match: () => true });
+ expect(all.files.length).toBe(1);
+ });
+
test("data URL parsing", () => {
expect(parseImageDataUrl("data:image/png;base64,AQID")).toEqual({ mime: "image/png", bytes: Buffer.from([1, 2, 3]) });
expect(parseImageDataUrl("data:text/plain;base64,AQID")).toBeNull();
diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.ts
index e6f77a92..fe3d77ef 100644
--- a/apps/server/src/computer-use/portable-host.test.ts
+++ b/apps/server/src/computer-use/portable-host.test.ts
@@ -24,12 +24,12 @@ function fakeNative() {
const listeners = new Set<(v: Record<string, unknown>) => void>();
const calls: Array<{ method: string; params: Record<string, unknown> }> = [];
let generation = 0;
- const state: { occluded: boolean; synthetic: boolean; retitle: ((title: string) => string) | null } = { occluded: false, synthetic: false, retitle: null };
+ const state: { occluded: boolean; synthetic: boolean; retitle: ((title: string) => string) | null; texts: Array<{ x: number; y: number; width: number; height: number; text: string }> | null } = { occluded: false, synthetic: false, retitle: null, texts: null };
const frame = (w: Window): Frame => {
const width = Math.min(w.bounds.width, 640), height = Math.round((width * w.bounds.height) / w.bounds.width);
const data = new Uint8Array(width * height * 4).fill(shade);
for (let i = 3; i < data.length; i += 4) data[i] = 255;
- return { width, height, data, secure: [{ x: 10, y: 10, width: 40, height: 10 }], caret: null };
+ return { width, height, data, secure: [{ x: 10, y: 10, width: 40, height: 10 }], caret: null, ...(state.texts ? { texts: state.texts } : {}) };
};
const native = {
ready: Promise.resolve({ supported: true }),
@@ -220,14 +220,18 @@ describe("portable host: observe and act", () => {
const first = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Type the greeting", action: { type: "type", text: "hi" } });
expect(first.state).toMatchObject({ ok: true, approval: { asked: true } });
expect(r.acts.length).toBe(1);
- // The model retries the same act on a fresh observation: no second question within 30 s.
+ // The screen changed meanwhile (shade 90), so the same act on the new image asks again ...
const again = await call(c, "computer_act", { session_id: session, observation_id: (first.state.observation as { id: string }).id, request_id: "b", intent: "Type the greeting", action: { type: "type", text: "hi" } });
- expect(again.state).toMatchObject({ ok: true, approval: { asked: false } });
- expect(r.acts.length).toBe(1);
+ expect(again.state).toMatchObject({ ok: true, approval: { asked: true } });
+ expect(r.acts.length).toBe(2);
+ // ... and a retry on the same image within 30 s does not.
+ const same = await call(c, "computer_act", { session_id: session, observation_id: (again.state.observation as { id: string }).id, request_id: "c", intent: "Type the greeting", action: { type: "type", text: "hi" } });
+ expect(same.state).toMatchObject({ ok: true, approval: { asked: false } });
+ expect(r.acts.length).toBe(2);
// Another act asks.
- const other = await call(c, "computer_act", { session_id: session, observation_id: (again.state.observation as { id: string }).id, request_id: "c", intent: "Type more", action: { type: "type", text: "there" } });
+ const other = await call(c, "computer_act", { session_id: session, observation_id: (same.state.observation as { id: string }).id, request_id: "d", intent: "Type more", action: { type: "type", text: "there" } });
expect(other.state).toMatchObject({ ok: true, approval: { asked: true } });
- expect(r.acts.length).toBe(2);
+ expect(r.acts.length).toBe(3);
await host.close();
});
@@ -245,6 +249,31 @@ describe("portable host: observe and act", () => {
await host.close();
});
+ test("Normal: the text the host read at the point decides (risky target, secure field); without text, a click after typing asks", async () => {
+ const r = rig("normal");
+ // A host that reads text (macOS): a "Sign in" button at 300,200 (native pixels; the image is 640 wide, window 2560).
+ r.native.state.texts = [{ x: 300, y: 200, width: 80, height: 30, text: "Sign in" }];
+ const { host, c, session, obs } = await opened(r);
+ const sign = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Press the button", action: { type: "click", x: 300, y: 200 } });
+ expect(sign.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["risky_target"] } });
+ const secure = await call(c, "computer_act", { session_id: session, observation_id: (sign.state.observation as { id: string }).id, request_id: "b", intent: "Fill in the field", action: { type: "type", text: "abc" } });
+ expect(secure.state).toMatchObject({ ok: true, approval: { asked: false } });
+ await host.close();
+ // A host that reads no text (X11, Windows): type, then a click asks.
+ const x = rig("normal");
+ const o = await opened(x);
+ const typed = await call(o.c, "computer_act", { session_id: o.session, observation_id: o.obs.state.observation?.id, request_id: "t", intent: "Type the name", action: { type: "type", text: "Ada" } });
+ expect(typed.state).toMatchObject({ ok: true, approval: { asked: false } });
+ const click = await call(o.c, "computer_act", { session_id: o.session, observation_id: (typed.state.observation as { id: string }).id, request_id: "k", intent: "Go to the next field", action: { type: "click", x: 400, y: 300 } });
+ expect(click.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["submit_after_typing"] } });
+ const again = await call(o.c, "computer_act", { session_id: o.session, observation_id: (click.state.observation as { id: string }).id, request_id: "k2", intent: "Click again", action: { type: "click", x: 400, y: 300 } });
+ expect(again.state).toMatchObject({ ok: true, approval: { asked: false } });
+ expect(await o.host.physicalInput()).toBe(true);
+ x.native.state.synthetic = true;
+ expect(await o.host.physicalInput()).toBe(false);
+ await o.host.close();
+ });
+
test("person input pauses; only Continue resumes; first act after takeover asks in Normal", async () => {
const r = rig("normal");
const { host, c, session, obs } = await opened(r);
diff --git a/apps/server/src/computer-use/routes.ts b/apps/server/src/computer-use/routes.ts
index b364d2ab..1e62640a 100644
--- a/apps/server/src/computer-use/routes.ts
+++ b/apps/server/src/computer-use/routes.ts
@@ -13,6 +13,9 @@ import {
computerUseDir,
computerUseSettingFor,
type createChatTracker,
+ expectedHostCommand,
+ hostSecretMatches,
+ sameHostCommand,
mediaSpoolDir,
permissionModeName,
readHostFile,
@@ -69,18 +72,27 @@ export function registerComputerUseRoutes(deps: ComputerUseRouteDeps): void {
return jsonResponse(await state());
});
+ // Host and event: only Electron main (its per-launch secret, held in this process); the client token is
+ // readable by the agent's shell, so it is not enough to replace the server's command or forge events.
+ const fromMain = (request: Request) => hostSecretMatches(request.headers.get("x-omnirush-computer-use-secret"));
+
addRoute(routes, "POST", "/computer-use/host", "client", async (ctx) => {
+ if (!fromMain(ctx.request)) return jsonResponse({ ok: false, error: "forbidden" }, 403);
const body = await deps.readJsonBody(ctx.request);
const command = Array.isArray(body.command) && body.command.every((part) => typeof part === "string") ? (body.command as string[]) : null;
const environment = typeof body.environment === "object" && body.environment !== null && !Array.isArray(body.environment)
? Object.fromEntries(Object.entries(body.environment as Record<string, unknown>).filter((entry): entry is [string, string] => typeof entry[1] === "string"))
: {};
- const changed = writeHostFile(computerUseDir(deps.stateDir()), command && command.length ? { command, environment } : null);
+ const host = command && command.length ? { command, environment } : null;
+ // Only the app's own relay command (expectedHostCommand), or none.
+ if (host && !sameHostCommand(host, expectedHostCommand())) return jsonResponse({ ok: false, error: "unexpected_command" }, 400);
+ const changed = writeHostFile(computerUseDir(deps.stateDir()), host);
if (changed) void deps.applyEngine().catch(() => undefined);
return jsonResponse({ ok: true, changed });
});
addRoute(routes, "POST", "/computer-use/event", "client", async (ctx) => {
+ if (!fromMain(ctx.request)) return jsonResponse({ ok: false, error: "forbidden" }, 403);
const body = await deps.readJsonBody(ctx.request);
const event = sessionEventOf(body.event);
if (!event) return jsonResponse({ ok: false, error: "invalid_event" }, 400);
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256
index f1c04537..47262c32 100644
--- a/apps/server/src/computer-use/shared/PARITY.sha256
+++ b/apps/server/src/computer-use/shared/PARITY.sha256
@@ -1,13 +1,13 @@
# Computer use (contract v2, policy, media refs, image codecs, Windows/X11 host): the same files in omnirush-cli (src/computer-use/) and omnirush-gui (apps/server/src/computer-use/shared/).
# A change to one must be ported to the other and this list updated in both.
-7c23cd3e3a245f277569eafcb39c4f09ee61aafeb28b9c53903f86702c9cfccb contract.mjs
-07fe5681c685a1f4883ecfe3e6528c1d70cf68003f690079297f887990879d80 contract.d.mts
-c601c77fd65561657b103b6ec3c5e4e822d38d4cf9d535da368cc2466698a9d5 policy.mjs
-103c58c3908bebab415e5f1878abf752a2b07498f502855298a72cb58225798f policy.d.mts
+36b49e5c1a90d4eb9b7d8c8fda207e6385c7ec0f8ba633e96f6726fc92ab2f53 contract.mjs
+aef0973f520f56ab8ad2699c14428bb2dd7ddb4b00dff823e4598e64d0c4a35b contract.d.mts
+a147eeabed93c867fc12c8e205162dd4812a005de8176bbd005bc0cd8bc8bc5c policy.mjs
+57fec686eb98d9a80812355ea2c7bbe18cc8d0824934b5c8e9cdfc5669b0d757 policy.d.mts
a96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs
5eac31868aedd3e1d1e71b445f9a55d01316da7397f27f6db11472453bee4588 image.d.mts
-98ffcc70c69299917c9804c32b786e58c1677edf7ea0732a3fe12b9f6781ba29 media.mjs
-c12540410a558f4b48960c75b7a09e0660d6de679a6e49e2542427530fd9a3d4 media.d.mts
-a9d1df37870802ffdca408a29735d1ce2a629e93435bc46ff36a0c5f8108869a native-worker.mjs
-52d7c84fbb0ee09ff74bcf2f1b19c85745211b883140efaff69ca80880735631 portable-host.mjs
-444f1cdfd00eedc49626f7ce62a64fa3ca75b7c389abbe0b22a3bb3bb822711c portable-host.d.mts
+e47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjs
+eabdf78c56733577d05aad490d9d7d5ece6c22e9b3ea9e6e9803d1a4a546fdd4 media.d.mts
+04bc6a36b9ce3409d0c7a5775b5fb6ea0797aa7ac304d83da249d19af9aca1be native-worker.mjs
+c7f42644d0e3b575e6896d705afeae88ec92b3f603dcc310854571fd6fa2f18d portable-host.mjs
+2ed7be072df62d50e1b569e75b695d18ae0d0de6dabe5375bc494b859cf78710 portable-host.d.mts
diff --git a/apps/server/src/computer-use/shared/contract.d.mts b/apps/server/src/computer-use/shared/contract.d.mts
index 300c696d..5e4d14a4 100644
--- a/apps/server/src/computer-use/shared/contract.d.mts
+++ b/apps/server/src/computer-use/shared/contract.d.mts
@@ -126,3 +126,5 @@ export function mcpResult(state: Record<string, unknown>, image?: { data: Uint8A
export function parseResultText(text: unknown): Record<string, unknown> | null;
export function isComputerTool(name: unknown): boolean;
export function osName(platform?: string): string;
+export function keyFromKeysymName(name: unknown): string | null;
+export function keyFromVirtualKey(vk: unknown): string | null;
diff --git a/apps/server/src/computer-use/shared/contract.mjs b/apps/server/src/computer-use/shared/contract.mjs
index 69125131..a9d4d43f 100644
--- a/apps/server/src/computer-use/shared/contract.mjs
+++ b/apps/server/src/computer-use/shared/contract.mjs
@@ -408,6 +408,32 @@ export function isComputerTool(name) {
return typeof name === "string" && name.startsWith(TOOL_PREFIX) && TOOLS.includes(name.slice(TOOL_PREFIX.length));
}
+const KEYSYM_NAMES = { Return: "Enter", KP_Enter: "Enter", Escape: "Escape", BackSpace: "Backspace", Tab: "Tab", ISO_Left_Tab: "Tab", space: "Space", Delete: "Delete", Insert: "Insert", Left: "Left", Up: "Up", Right: "Right", Down: "Down", Home: "Home", End: "End", Prior: "PageUp", Next: "PageDown" };
+const VK_NAMES = { 13: "Enter", 27: "Escape", 8: "Backspace", 9: "Tab", 32: "Space", 46: "Delete", 45: "Insert", 37: "Left", 38: "Up", 39: "Right", 40: "Down", 36: "Home", 35: "End", 33: "PageUp", 34: "PageDown" };
+
+/**
+ * A pressed key as a terminal sees it, from an X11 keysym name: one lower-case
+ * letter or digit for printable keys ("a", "y", "1"), else the canonical name
+ * ("Enter", "Escape", "F5" ...); null for modifiers and the rest.
+ */
+export function keyFromKeysymName(name) {
+ if (typeof name !== "string" || !name) return null;
+ if (/^[A-Za-z0-9]$/.test(name)) return name.toLowerCase();
+ if (KEYSYM_NAMES[name]) return KEYSYM_NAMES[name];
+ const fn = /^F([1-9]|1[0-9]|2[0-4])$/.exec(name);
+ return fn ? name : null;
+}
+
+/** The same from a Windows virtual-key code (0x41-0x5A letters, 0x30-0x39 digits, 0x70- F keys). */
+export function keyFromVirtualKey(vk) {
+ if (!Number.isInteger(vk)) return null;
+ if (vk >= 0x41 && vk <= 0x5a) return String.fromCharCode(vk + 32);
+ if (vk >= 0x30 && vk <= 0x39) return String.fromCharCode(vk);
+ if (vk >= 0x60 && vk <= 0x69) return String.fromCharCode(vk - 0x30);
+ if (vk >= 0x70 && vk <= 0x87) return `F${vk - 0x6f}`;
+ return VK_NAMES[vk] ?? null;
+}
+
/** The OS name used in results and events. */
export function osName(platform = process.platform) {
return platform === "darwin" ? "macos" : platform === "win32" ? "windows" : platform === "linux" ? "linux" : platform;
diff --git a/apps/server/src/computer-use/shared/media.d.mts b/apps/server/src/computer-use/shared/media.d.mts
index 1566c9a5..a5ac9080 100644
--- a/apps/server/src/computer-use/shared/media.d.mts
+++ b/apps/server/src/computer-use/shared/media.d.mts
@@ -66,7 +66,7 @@ export function parseImageDataUrl(url: unknown): { mime: string; bytes: Buffer }
export function thumbnailOf(bytes: Uint8Array, mime: string): Buffer | null;
export function createMediaSpool(dir: string): MediaSpool;
export function createMediaIndex(file: string): MediaIndex;
-export function rewriteMessageMedia<T>(message: T, ctx: { spool: MediaSpool; index: MediaIndex; limits?: { media_full_bytes?: number; media_model_bytes?: number } | null; thumbBudget?: number; chat?: string }): { message: T; changed: boolean; files: MediaFile[] };
+export function rewriteMessageMedia<T>(message: T, ctx: { spool: MediaSpool; index: MediaIndex; limits?: { media_full_bytes?: number; media_model_bytes?: number } | null; thumbBudget?: number; chat?: string; match?: (part: Record<string, unknown>) => boolean }): { message: T; changed: boolean; files: MediaFile[] };
export function mediaRefsOf(message: unknown): Array<{ sha: string; ext: string }>;
export function fileBytes(file: string): number;
export function pruneImages(messages: unknown[], options?: { keep?: number; drop?: number; match?: (part: Record<string, unknown>) => boolean }): number;
diff --git a/apps/server/src/computer-use/shared/media.mjs b/apps/server/src/computer-use/shared/media.mjs
index a640617c..77ca0397 100644
--- a/apps/server/src/computer-use/shared/media.mjs
+++ b/apps/server/src/computer-use/shared/media.mjs
@@ -20,7 +20,7 @@ import { createHash, randomUUID } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
import path from "node:path";
-import { DEFAULT_LIMITS } from "./contract.mjs";
+import { DEFAULT_LIMITS, isComputerTool } from "./contract.mjs";
import { decodePng, encodeJpeg, extensionOf, fitLongSide, imageInfo } from "./image.mjs";
export const MEDIA_ROOT = "__media__";
@@ -229,8 +229,9 @@ function budgets(limits) {
/**
* Replaces image data URLs in the tool parts of one engine message (in place
* on a structured copy; the input is not changed) with media references.
- * `ctx`: { spool, index, limits, thumbBudget, chat } (chat: the chat's id, for
- * the spool's references). Returns
+ * `ctx`: { spool, index, limits, thumbBudget, chat, match } (chat: the chat's
+ * id, for the spool's references; match(part): which tool parts, default the
+ * built-in computer-use tools only). Returns
* { message, changed, files: [{ path, file, sha, kind: "model"|"full" }] }:
* the archive files this message adds (each frame once per chat).
* User file parts are left alone (the archive keeps those as attachments).
@@ -296,8 +297,10 @@ export function rewriteMessageMedia(message, ctx) {
}
return out ?? value;
};
+ // Computer use parts only by default: other tools' images (playwright ...) upload exactly as before.
+ const match = typeof ctx.match === "function" ? ctx.match : (part) => isComputerTool(part.tool);
const parts = message.parts.map((part) => {
- if (!isRecord(part) || part.type !== "tool" || !isRecord(part.state)) return part;
+ if (!isRecord(part) || part.type !== "tool" || !isRecord(part.state) || !match(part)) return part;
// Attachments first: they carry the chat's first thumbnail of a frame.
const attachments = Array.isArray(part.state.attachments) ? part.state.attachments : null;
const next = attachments && attachments.some((a) => isRecord(a) && typeof a.url === "string" && a.url.startsWith("data:image/"))
diff --git a/apps/server/src/computer-use/shared/native-worker.mjs b/apps/server/src/computer-use/shared/native-worker.mjs
index 4bfe132b..5721dd3e 100644
--- a/apps/server/src/computer-use/shared/native-worker.mjs
+++ b/apps/server/src/computer-use/shared/native-worker.mjs
@@ -16,6 +16,8 @@ import { readFileSync, readlinkSync } from "node:fs";
import { createRequire } from "node:module";
import path from "node:path";
+import { keyFromKeysymName, keyFromVirtualKey } from "./contract.mjs";
+
const require = createRequire(import.meta.url);
const koffi = require(process.env.OMNIRUSH_KOFFI || "koffi");
@@ -24,6 +26,19 @@ let generation = 0;
let backend;
let held = { keys: [], buttons: [] };
const lastInput = { synthetic: false, at: 0, kind: "" };
+// The latest physical presses (not injected): what was pressed and which app was in front, for the
+// hosts' "the person answered" checks. A short ring, so presses during an open question are kept.
+const presses = [];
+function notePress(entry) {
+ Object.assign(lastInput, { synthetic: entry.synthetic, at: entry.at, kind: entry.kind });
+ if (entry.synthetic) return;
+ presses.push({ at: entry.at, kind: entry.kind, key: entry.key ?? null, foreground_pid: entry.foreground_pid ?? null });
+ if (presses.length > 16) presses.shift();
+}
+function inputSource() {
+ const physical = presses.at(-1) ?? null;
+ return { ...lastInput, key: physical?.key ?? null, foreground_pid: physical?.foreground_pid ?? null, physical_at: physical?.at ?? 0, presses: presses.slice() };
+}
const changed = () => { generation += 1; post({ event: "person-input", generation }); };
const fail = (code, message) => { throw Object.assign(new Error(message), { code }); };
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -151,7 +166,11 @@ function windows() {
const value = koffi.decode(lParam, type);
if (Number(value.dwExtraInfo) !== marker) {
const down = [0x100, 0x104, 0x201, 0x204, 0x207].includes(Number(wParam));
- if (down) Object.assign(lastInput, { synthetic: (value.flags & injectedFlag) !== 0, at: Date.now(), kind: type === hookKey ? "key" : "pointer" });
+ if (down) {
+ const fg = [0];
+ try { pidOf(foreground(), fg); } catch { /* unknown */ }
+ notePress({ synthetic: (value.flags & injectedFlag) !== 0, at: Date.now(), kind: type === hookKey ? "key" : "pointer", key: type === hookKey ? keyFromVirtualKey(Number(value.vkCode)) : null, foreground_pid: fg[0] || null });
+ }
if (interrupting) { changed(); release(); }
}
}
@@ -479,6 +498,9 @@ function x11() {
const selectEvents = xi.func("int XISelectEvents(void *display, ulong window, CU2_XIEventMask *masks, int count)");
const major = [2], minor = [0];
if (xi.func("int XIQueryVersion(void *display, _Inout_ int *major, _Inout_ int *minor)")(display, major, minor) !== 0) fail("input_unavailable", "XInput2 is required to notice your input.");
+ const keycodeKeysym = x.func("ulong XkbKeycodeToKeysym(void *display, uint8_t keycode, int group, int level)");
+ const keysymString = x.func("str XKeysymToString(ulong keysym)");
+ const keysymName = (sym) => (sym ? keysymString(sym) : null);
const testDevices = new Set();
const refreshDevices = () => { const count = [0], ptr = queryDevices(display, 0, count); testDevices.clear(); if (ptr) { for (const d of koffi.decode(ptr, deviceType, count[0])) if (d.name.includes("XTEST")) testDevices.add(d.deviceid); freeDevices(ptr); } };
refreshDevices();
@@ -499,7 +521,15 @@ function x11() {
const raw = koffi.decode(cookie.data, rawType);
const ours = injecting && testDevices.has(raw.sourceid);
if (ours) continue;
- if (cookie.evtype === 13 || cookie.evtype === 15) Object.assign(lastInput, { synthetic: testDevices.has(raw.sourceid), at: Date.now(), kind: cookie.evtype === 13 ? "key" : "pointer" });
+ if (cookie.evtype === 13 || cookie.evtype === 15) {
+ const synthetic = testDevices.has(raw.sourceid);
+ let key = null, fg = null;
+ if (!synthetic) {
+ try { if (cookie.evtype === 13) key = keyFromKeysymName(keysymName(keycodeKeysym(display, raw.detail, 0, 0))); } catch { /* unknown */ }
+ try { fg = Number(prop(activeWindow(), "_NET_WM_PID")[0] ?? 0) || null; } catch { /* unknown */ }
+ }
+ notePress({ synthetic, at: Date.now(), kind: cookie.evtype === 13 ? "key" : "pointer", key, foreground_pid: fg });
+ }
if (interrupting) changed();
}
} finally { freeEventData(display, cookie); }
@@ -859,7 +889,7 @@ process.on("message", async (message) => {
post({ id, result: {} });
return;
}
- if (method === "input-source") { backend.poll(); post({ id, result: { ...lastInput } }); return; }
+ if (method === "input-source") { backend.poll(); post({ id, result: inputSource() }); return; }
if (busy) fail("busy", "Desktop operations must be sequential.");
busy = true; ownsBusy = true;
backend.poll();
diff --git a/apps/server/src/computer-use/shared/policy.d.mts b/apps/server/src/computer-use/shared/policy.d.mts
index 48fe36a5..250a6f1a 100644
--- a/apps/server/src/computer-use/shared/policy.d.mts
+++ b/apps/server/src/computer-use/shared/policy.d.mts
@@ -56,7 +56,7 @@ export function looksLikeCredential(value: unknown): boolean;
export function riskyChord(action: unknown): "paste" | "close_quit_delete" | null;
export function documentOf(title: unknown, appName?: unknown): string;
-export type AskReasonCode = "careful" | "risky_intent" | "risky_target" | "secure_field" | "credential_text" | "paste" | "close_quit_delete" | "other_window" | "document_changed" | "after_takeover";
+export type AskReasonCode = "careful" | "risky_intent" | "risky_target" | "secure_field" | "credential_text" | "paste" | "close_quit_delete" | "other_window" | "document_changed" | "after_takeover" | "submit_after_typing";
export type ApprovalInput = {
mode?: PermissionMode | string;
action?: Action | Record<string, unknown>;
@@ -65,6 +65,8 @@ export type ApprovalInput = {
window?: { id?: string | number; app_id?: string; title?: string; app_name?: string; url_host?: string };
approved?: { window_id?: string | number; app_id?: string; title?: string; url_host?: string };
after_takeover?: boolean;
+ /** The agent's previous act in this window typed text (and no click or Enter since). */
+ typed_before?: boolean;
};
export function actApproval(input?: ApprovalInput): { ask: boolean; reasons: Array<{ code: AskReasonCode; detail?: string }> };
export const ASK_REASONS: Readonly<Record<AskReasonCode, string>>;
diff --git a/apps/server/src/computer-use/shared/policy.mjs b/apps/server/src/computer-use/shared/policy.mjs
index 31d6bbb8..7297fa94 100644
--- a/apps/server/src/computer-use/shared/policy.mjs
+++ b/apps/server/src/computer-use/shared/policy.mjs
@@ -347,7 +347,9 @@ export function documentOf(title, appName) {
* indicator and Stop still apply).
* `input`: { mode, action, intent, target: { text, secure, url_host },
* window: { id, app_id, title, app_name, url_host }, approved: { window_id,
- * app_id, title, url_host }, after_takeover }.
+ * app_id, title, url_host }, after_takeover, typed_before } (target.text
+ * undefined: the host reads no text there; typed_before: the last act in this
+ * window was `type`).
* Returns { ask, reasons: [{ code, detail? }] }.
*/
export function actApproval(input = {}) {
@@ -363,6 +365,9 @@ export function actApproval(input = {}) {
const targetWords = riskyWords(target.text);
if (targetWords.length) reasons.push({ code: "risky_target", detail: targetWords.join(", ") });
}
+ // No text the host could read at the target: a click or Enter right after typing in this window
+ // may submit what was typed (a login, a message, a payment), so it asks.
+ if (typeof target.text !== "string" && input.typed_before === true && (["click", "double_click"].includes(action.type) || (action.type === "key" && String(action.key).toLowerCase() === "enter"))) reasons.push({ code: "submit_after_typing" });
if (action.type === "type") {
if (target.secure === true) reasons.push({ code: "secure_field" });
if (looksLikeCredential(action.text)) reasons.push({ code: "credential_text" });
@@ -391,6 +396,7 @@ export const ASK_REASONS = Object.freeze({
other_window: "It would act in another window.",
document_changed: "The window now shows a different page or document",
after_takeover: "First action after you took over.",
+ submit_after_typing: "It would click or press Enter right after typing, which may send what was typed.",
});
/**
diff --git a/apps/server/src/computer-use/shared/portable-host.d.mts b/apps/server/src/computer-use/shared/portable-host.d.mts
index 01a993de..bfb78b6d 100644
--- a/apps/server/src/computer-use/shared/portable-host.d.mts
+++ b/apps/server/src/computer-use/shared/portable-host.d.mts
@@ -79,6 +79,8 @@ export type PortableHost = {
control(input: { session_id: string; action: "stop" | "takeover" | "continue" | "hide" | "show" }): Promise<void>;
stopAll(by?: SessionEvent["by"], event?: SessionEvent["event"]): Promise<void>;
syntheticInput(withinMs?: number): Promise<boolean>;
+ /** A physical key or button press within `withinMs` (not injected, not a remote-debugging click). */
+ physicalInput(withinMs?: number, options?: { foregroundPid?: number | null }): Promise<boolean>;
watchInput(): Promise<void>;
state(): HostSessionState[];
close(): Promise<void>;
diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjs
index 5ca3bff3..0fe3c51d 100644
--- a/apps/server/src/computer-use/shared/portable-host.mjs
+++ b/apps/server/src/computer-use/shared/portable-host.mjs
@@ -272,11 +272,15 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
try { spool.put({ bytes, mime, width: model.width, height: model.height, full, thumb: encodeJpeg(fitLongSide(model, 320), { quality: 70 }), ocr, masked }); } catch { /* the archive copy is best effort */ }
}
const ratio = model.width / raw.width;
- const caret = raw.caret ? { x: raw.caret.x * ratio, y: raw.caret.y * ratio, width: raw.caret.width * ratio, height: raw.caret.height * ratio } : null;
+ const scaled = (r) => ({ x: r.x * ratio, y: r.y * ratio, width: r.width * ratio, height: r.height * ratio });
+ const caret = raw.caret ? scaled(raw.caret) : null;
+ // What the host can read on screen (macOS: AX / text recognition), for the per-act rules only: never sent to the model.
+ const texts = Array.isArray(raw.texts) ? raw.texts.filter((t) => t && typeof t.text === "string" && [t.x, t.y, t.width, t.height].every(Number.isFinite)).slice(0, 2000).map((t) => ({ ...scaled(t), text: t.text.slice(0, 200) })) : null;
+ const secureFields = (Array.isArray(raw.secure_fields) ? raw.secure_fields : raw.secure ?? []).filter((r) => r && [r.x, r.y, r.width, r.height].every(Number.isFinite)).map(scaled);
const id = randomUUID();
const value = {
id, at: Date.now(), window: w, sha, mime, bytes,
- model: { width: model.width, height: model.height }, grid: luminanceGrid(model), caret,
+ model: { width: model.width, height: model.height }, grid: luminanceGrid(model), caret, texts, secureFields,
ref: { id, width: model.width, height: model.height, sha256: sha, mime, ...(full ? { full: { width: full.width, height: full.height, sha256: full.sha256 } } : {}), ocr, masked },
};
s.lastFrame = { sha, width: model.width, height: model.height, mime, data_url: `data:${mime};base64,${bytes.toString("base64")}`, at: now() };
@@ -307,6 +311,17 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
return mcpResult(observeResult({ session_id: s.id, observation: obs.ref, window: windowRef(w), screen: s.screen, scale: w.bounds.width / obs.model.width, dpi: w.dpi, monitor_id: w.monitor, cursor: await native.call("cursor").catch(() => null), os, expires_in_ms: l.observation_ttl_seconds * 1000, actions_left: Math.max(0, l.actions_per_session - s.actions), seconds_left: Math.max(0, Math.ceil((s.expiresAt - Date.now()) / 1000)) }), includeImage ? { data: obs.bytes, mime: obs.mime } : null);
}
+ /** The on-screen text at (or within 24 px of) the action's point, and whether it is in a secure field. Unknown text: undefined. */
+ function targetAt(observed, action) {
+ const p = action.type === "drag" ? action.path.at(-1) : action.x !== undefined ? { x: action.x, y: action.y } : null;
+ if (!p) return {};
+ const inside = (r, pad) => p.x >= r.x - pad && p.x <= r.x + r.width + pad && p.y >= r.y - pad && p.y <= r.y + r.height + pad;
+ const secure = (observed.secureFields ?? []).some((r) => inside(r, 0));
+ if (!observed.texts) return secure ? { secure } : {};
+ const near = observed.texts.filter((t) => inside(t, 24)).sort((a, b) => (inside(a, 0) ? 0 : 1) - (inside(b, 0) ? 0 : 1)).slice(0, 4);
+ return { text: near.map((t) => t.text).join(" "), ...(secure ? { secure } : {}) };
+ }
+
async function actTool(s, args) {
const l = limits();
const started = Date.now();
@@ -349,11 +364,12 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
if (compare.stale) return finish({ ...common, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });
}
// Per-action approval by permission mode (host channel, never the engine's).
- const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: {}, window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover });
+ const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: targetAt(observed, action), window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover, typed_before: s.typedIn === String(current.id) });
let approval = { asked: false };
if (decision.ask) {
// The same action with the same intent, allowed a moment ago (a retry after a fresh observation), is not asked again.
- const key = JSON.stringify([action, args.intent]);
+ // Keyed to the image too: the same screen, the same action, the same reason.
+ const key = JSON.stringify([action, args.intent, observed.sha]);
const remembered = s.allowed.get(key);
if (remembered && remembered > Date.now()) {
approval = { asked: false, reasons: decision.reasons.map((r) => r.code), by: "person" };
@@ -390,6 +406,9 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
return finish({ ...common, approval, point, path, cursor_before: cursorBefore, ok: false, code, message: error.message, may_have_acted: action.type !== "wait", next: "computer_session_status" });
}
s.lastOperation = Date.now();
+ // "Submit after typing" (Normal, when the host reads no text): typing arms it, a click or Enter after it asks.
+ if (action.type === "type") s.typedIn = String(current.id);
+ else if (["click", "double_click"].includes(action.type) || (action.type === "key" && action.key === "Enter")) s.typedIn = null;
const cursorAfter = await native.call("cursor").catch(() => null);
const done = { ...common, approval, point, path, cursor_before: cursorBefore, cursor_after: cursorAfter, ok: true, may_have_acted: action.type !== "wait" };
if (!args.observe_after) return finish({ ...done, next: "computer_observe" });
@@ -626,6 +645,17 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
}
/** Whether the latest press (key or button) within `withinMs` was synthetic (injected by a program). */
+ /**
+ * Whether a physical key or button press (not injected by a program, not a remote-debugging
+ * click that makes no OS input) came within `withinMs`: the person's own answer.
+ */
+ async function physicalInput(withinMs = 5000, { foregroundPid = null } = {}) {
+ const value = await native.call("input-source").catch(() => null);
+ if (!value || !value.at || value.synthetic || Date.now() - value.at >= withinMs) return false;
+ // In the answering app's window (when the worker knows which app was in front).
+ return !(foregroundPid && Number.isInteger(value.foreground_pid) && value.foreground_pid !== foregroundPid);
+ }
+
async function syntheticInput(withinMs = 5000) {
const value = await native.call("input-source").catch(() => null);
return Boolean(value && value.synthetic && Date.now() - value.at < withinMs);
@@ -642,5 +672,5 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
}
update();
- return { get availability() { return availability; }, os, connect, listen, control, stopAll, syntheticInput, watchInput, state: publicState, close };
+ return { get availability() { return availability; }, os, connect, listen, control, stopAll, syntheticInput, physicalInput, watchInput, state: publicState, close };
}
diff --git a/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts b/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts
index 7bd63483..51a98e3e 100644
--- a/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts
+++ b/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts
@@ -65,14 +65,14 @@ function withScreenBashRules(permission: Record<string, unknown>): Record<string
}
/** Adds the left-out `computer-use` entry to the project's MCP report (Settings > MCP). */
-function reportReserved(root: string): void {
+function reportReserved(root: string, names: string[]): void {
const dir = process.env.OMNIRUSH_MCP_REPORT_DIR;
if (!dir) return;
try {
const file = reportPath(dir, root);
const report: unknown = JSON.parse(readFileSync(file, "utf8"));
if (!isRecord(report) || !Array.isArray(report.decisions)) return;
- report.decisions = [...report.decisions.filter((d) => !(isRecord(d) && d.name === SERVER_ID)), { name: SERVER_ID, status: "blocked", code: "reserved", reason: "This name belongs to the app's built-in computer use; this entry is left out." }];
+ report.decisions = [...report.decisions.filter((d) => !(isRecord(d) && names.includes(String(d.name)))), ...names.map((name) => ({ name, status: "blocked", code: "reserved", reason: "This name belongs to the app's built-in computer use; this entry is left out." }))];
const temp = `${file}.${process.pid}.cu.tmp`;
writeFileSync(temp, JSON.stringify(report), { mode: 0o600 });
renameSync(temp, file);
@@ -93,7 +93,24 @@ function withoutSubagentComputerTools(target: Record<string, unknown>): void {
target.agent = agents;
}
-export default async function omnirushMcpPolicy(input?: { directory?: string }) {
+/** Config names that are, or would make tool names of, the built-in server (`computer-use`, `computer-use_x` → `computer-use_x_act`). */
+function reservedNames(mcp: Record<string, unknown>): string[] {
+ return Object.keys(mcp).filter((name) => name.toLowerCase().startsWith(SERVER_ID));
+}
+
+type SessionClient = { session?: { get?: (options: { path: { id: string } }) => Promise<{ data?: { parentID?: unknown } } | undefined> } };
+
+export default async function omnirushMcpPolicy(input?: { directory?: string; client?: SessionClient }) {
+ // Sub-agents (sessions with a parent) never use computer use (one host lease per app).
+ const parents = new Map<string, boolean>();
+ const isSubagent = async (sessionID: unknown): Promise<boolean> => {
+ if (typeof sessionID !== "string" || !sessionID || !input?.client?.session?.get) return false;
+ if (!parents.has(sessionID)) {
+ const found = await input.client.session.get({ path: { id: sessionID } }).catch(() => undefined);
+ parents.set(sessionID, Boolean(found?.data?.parentID));
+ }
+ return parents.get(sessionID) === true;
+ };
const root = path.resolve(input?.directory || process.cwd());
const hooks = createMcpPolicyHooks({
directory: root,
@@ -113,14 +130,14 @@ export default async function omnirushMcpPolicy(input?: { directory?: string })
}
}
// The id `computer-use` belongs to the app's built-in server: a config entry with it is left out (and reported).
- const reserved = target && isRecord(target.mcp) && Object.hasOwn(target.mcp, SERVER_ID);
- if (target && reserved && isRecord(target.mcp)) {
- const { [SERVER_ID]: _dropped, ...rest } = target.mcp;
- target.mcp = rest;
+ const reserved = target && isRecord(target.mcp) ? reservedNames(target.mcp) : [];
+ if (target && reserved.length && isRecord(target.mcp)) {
+ const mcp = target.mcp;
+ target.mcp = Object.fromEntries(Object.entries(mcp).filter(([name]) => !reserved.includes(name)));
}
// Fails closed: an error while applying the list leaves no servers at all.
await hooks.config(configInput, output);
- if (reserved) reportReserved(root);
+ if (reserved.length) reportReserved(root, reserved);
const builtin = target ? computerUseBuiltins(process.env[COMPUTER_USE_DIR_ENV])[SERVER_ID] : null;
if (target && builtin) {
target.mcp = { ...(isRecord(target.mcp) ? target.mcp : {}), [SERVER_ID]: builtin };
@@ -139,6 +156,11 @@ export default async function omnirushMcpPolicy(input?: { directory?: string })
"experimental.chat.messages.transform": async (_input: unknown, output: { messages?: unknown[] }) => {
if (Array.isArray(output?.messages)) pruneImages(output.messages, { match: (part: Record<string, unknown>) => isComputerTool(part.tool) });
},
- "tool.execute.before": hooks["tool.execute.before"],
+ "tool.execute.before": async (toolInput: { tool?: unknown; args?: unknown; sessionID?: unknown }, toolOutput?: { args?: unknown }) => {
+ if (typeof toolInput?.tool === "string" && toolInput.tool.startsWith(`${SERVER_ID}_`) && (await isSubagent(toolInput.sessionID))) {
+ throw new Error("Computer use is not available to sub-agents. Ask the main agent to use it.");
+ }
+ await hooks["tool.execute.before"](toolInput, toolOutput);
+ },
};
}