A desktop coding agent with free access to frontier models.
TypeScript
fix: computer use review fixes
ogsamrat6135cf9d1efe
diff --git a/apps/desktop/electron/computer-use-host.mjs b/apps/desktop/electron/computer-use-host.mjsindex 6e91b1cf..c9e0a1ee 100644--- a/apps/desktop/electron/computer-use-host.mjs+++ b/apps/desktop/electron/computer-use-host.mjs@@ -147,7 +147,9 @@ export function createMacNativeWorker({ executable, args = ["worker"], captureLo// Painted here on every capture (the host paints the same boxes again, which changes nothing), so// the host's stale check compares a masked observation with a masked fresh capture.paintBoxes?.(frame, secure);- return { width: frame.width, height: frame.height, data: frame.data, secure, caret: frame.caret ?? null, ocr: frame.ocr === "client" ? "client" : "none", credential_boxes: text.length, source: frame.source };+ // The recognized lines (boxes + text) for the host's per-act rules (risky targets); never sent to the model.+ const texts = Array.isArray(frame.text_boxes) ? frame.text_boxes.filter((l) => record(l) && typeof l.text === "string" && record(l.box)).map((l) => ({ x: l.box.x, y: l.box.y, width: l.box.width, height: l.box.height, text: l.text })) : null;+ return { width: frame.width, height: frame.height, data: frame.data, secure, secure_fields: Array.isArray(frame.secure) ? frame.secure : [], ...(frame.ocr === "client" && texts ? { texts } : {}), caret: frame.caret ?? null, ocr: frame.ocr === "client" ? "client" : "none", credential_boxes: text.length, source: frame.source };},kill() { child.kill(); },async close() { if (closed) return; await raw("shutdown", {}, 3000).catch(() => {}); closed = true; if (child.exitCode === null) child.kill(); },diff --git a/apps/desktop/electron/computer-use.mjs b/apps/desktop/electron/computer-use.mjsindex 4e46f341..34e13fb0 100644--- a/apps/desktop/electron/computer-use.mjs+++ b/apps/desktop/electron/computer-use.mjs@@ -12,6 +12,7 @@// chat's permission mode, are answered in the main window (desktop IPC from// its main frame, input checked for synthetic events), never by the agent.import { spawn } from "node:child_process";+import { randomBytes } from "node:crypto";import { existsSync } from "node:fs";import { createRequire } from "node:module";import path from "node:path";@@ -24,7 +25,22 @@ import { createComputerPreview } from "./computer-use-preview.mjs";const __dirname = path.dirname(fileURLToPath(import.meta.url));const require = createRequire(import.meta.url);const STATUS_CHANNEL = "omnirush:computer-use:status";+// A per-launch secret, held in this process only (the local server runs in this process and reads it here):+// the host and event routes take it, so a client token read from disk cannot replace the host or forge events.+const HOST_SECRET = randomBytes(32).toString("hex");+globalThis[Symbol.for("omnirush.computer-use.host-secret")] = HOST_SECRET;+const secretHeaders = { "x-omnirush-computer-use-secret": HOST_SECRET };+// The profile folder the local server expects the host in (computer-use/desktop.ts expectedHostCommand).+const profileDir = () => process.env.OMNIRUSH_DESKTOP_USER_DATA_DIR?.trim() || app.getPath("userData");const SYNTHETIC_MESSAGE = "Use your own mouse or keyboard to approve.";+// Answers need a physical press in this app's window. OMNIRUSH_COMPUTER_USE_TEST_ANSWERS=1 (rigs only: the+// app's own environment, which the agent cannot set for a running app) accepts remote-debugging clicks.+const testAnswers = process.env.OMNIRUSH_COMPUTER_USE_TEST_ANSWERS === "1";+async function personAnswered() {+ if (!host) return false;+ if (await host.syntheticInput()) return false;+ return testAnswers || host.physicalInput(5000, { foregroundPid: process.pid });+}let deps = null;let server = null; // the last /computer-use/state answer@@ -148,7 +164,7 @@ async function hostContext() {async function reportEvent(event) {const session = host?.state().find((s) => s.id === event.session_id);- await serverFetch("/computer-use/event", { method: "POST", body: JSON.stringify({ event, ...(session?.chat ? { chat: session.chat } : {}) }) }).catch(() => {});+ await serverFetch("/computer-use/event", { method: "POST", headers: secretHeaders, body: JSON.stringify({ event, ...(session?.chat ? { chat: session.chat } : {}) }) }).catch(() => {});}async function startHost() {@@ -174,11 +190,11 @@ async function startHost() {preview.bind(created);}host = created;- listener = await created.listen(path.join(app.getPath("userData"), "computer-use"));+ listener = await created.listen(path.join(profileDir(), "computer-use"));const relay = app.isPackaged ? path.join(process.resourcesPath, "computer-use", "relay.mjs") : path.join(__dirname, "computer-use-relay.mjs");const command = [process.execPath, relay, listener.discoveryPath];const environment = { ELECTRON_RUN_AS_NODE: "1" };- await serverFetch("/computer-use/host", { method: "POST", body: JSON.stringify({ command, environment }) });+ await serverFetch("/computer-use/host", { method: "POST", headers: secretHeaders, body: JSON.stringify({ command, environment }) });pushStatus();return created;})().catch((error) => { hostPromise = null; host = null; throw error; });@@ -187,7 +203,7 @@ async function startHost() {async function stopHost(by = "server") {const current = host;- await serverFetch("/computer-use/host", { method: "POST", body: JSON.stringify({ host: null }) }).catch(() => {});+ await serverFetch("/computer-use/host", { method: "POST", headers: secretHeaders, body: JSON.stringify({ host: null }) }).catch(() => {});if (!current) return;await current.stopAll(by).catch(() => {});for (const entry of pending.values()) entry.resolve({ decision: "deny" });@@ -241,14 +257,14 @@ async function computerUseAnswer(answer) {const entry = pending.get(answer?.request_id);if (!entry) throw new Error("This request has ended.");const allowing = answer.decision !== "deny";- if (allowing && host && await host.syntheticInput()) throw new Error(SYNTHETIC_MESSAGE);+ if (allowing && !(await personAnswered())) throw new Error(SYNTHETIC_MESSAGE);const decision = entry.request.kind === "open" ? (["chat", "once", "deny"].includes(answer.decision) ? answer.decision : "deny") : (answer.decision === "allow" ? "allow" : "deny");entry.resolve({ decision, ...(typeof answer.window_id === "string" ? { window_id: answer.window_id } : {}) });}async function computerUseControl(value) {if (!host) throw new Error("Computer use is not running.");- if (value?.action === "continue" && await host.syntheticInput()) throw new Error(SYNTHETIC_MESSAGE);+ if (value?.action === "continue" && !(await personAnswered())) throw new Error(SYNTHETIC_MESSAGE);await host.control({ session_id: value?.session_id, action: value?.action });pushStatus();}diff --git a/apps/server/src/computer-use/contract-policy.test.ts b/apps/server/src/computer-use/contract-policy.test.tsindex 7d0a83a7..6d7f8182 100644--- a/apps/server/src/computer-use/contract-policy.test.ts+++ b/apps/server/src/computer-use/contract-policy.test.ts@@ -8,6 +8,8 @@ import {INSTRUCTIONS,isComputerTool,isFailure,+ keyFromKeysymName,+ keyFromVirtualKey,mapPoint,mcpResult,modelSize,@@ -62,6 +64,13 @@ describe("contract v2: tools", () => {});});+describe("contract v2: pressed keys as a terminal sees them", () => {+ test("X11 keysym names and Windows virtual keys", () => {+ expect(["a", "Y", "1", "Return", "Escape", "space", "F5", "Shift_L", "BackSpace"].map(keyFromKeysymName)).toEqual(["a", "y", "1", "Enter", "Escape", "Space", "F5", null, "Backspace"]);+ expect([0x41, 0x5a, 0x30, 0x0d, 0x1b, 0x74, 0x10, 0x63].map(keyFromVirtualKey)).toEqual(["a", "z", "0", "Enter", "Escape", "F5", null, "3"]);+ });+});+describe("contract v2: validation", () => {test("act needs intent, ids and an action", () => {const base = { session_id: "s1", observation_id: "o1", request_id: "r1", intent: "Open the File menu", action: { type: "click", x: 1, y: 1 } };@@ -241,6 +250,15 @@ describe("policy: approvals per permission mode", () => {expect(documentOf("● main.ts - Code", "Code")).toBe("main.ts");});+ test("Normal, no readable text at the target: a click or Enter right after typing asks", () => {+ expect(actApproval({ ...plain, target: {}, typed_before: true }).reasons).toEqual([{ code: "submit_after_typing" }]);+ expect(actApproval({ ...plain, action: { type: "key", key: "Enter" }, target: {}, typed_before: true }).ask).toBe(true);+ expect(actApproval({ ...plain, target: {}, typed_before: false }).ask).toBe(false);+ // With text the host read there, the target rules decide instead.+ expect(actApproval({ ...plain, target: { text: "Bold" }, typed_before: true }).ask).toBe(false);+ expect(actApproval({ ...plain, mode: "auto", target: {}, typed_before: true }).ask).toBe(false);+ });+test("shell screen capture asks; sub-agents get no computer tools", () => {expect(SCREEN_BASH_RULES["*xdotool*"]).toBe("ask");expect(SCREEN_BASH_RULES["*screencapture*"]).toBe("ask");diff --git a/apps/server/src/computer-use/desktop.test.ts b/apps/server/src/computer-use/desktop.test.tsindex 17cddb71..a5933e0a 100644--- a/apps/server/src/computer-use/desktop.test.ts+++ b/apps/server/src/computer-use/desktop.test.ts@@ -7,6 +7,9 @@ import path from "node:path";import omnirushMcpPolicy from "../opencode-plugins/omnirush-mcp-policy.js";import { MediaStore } from "../session-archive/media-store.js";+import type { Route } from "../routes/registry.js";+import { registerComputerUseRoutes } from "./routes.js";+import { expectedHostCommand, HOST_SECRET_KEY } from "./desktop.js";import { computerUseBuiltins, computerUseDir, computerUseSettingFor, createChatTracker, permissionModeName, rememberComputerUseSwitch, sessionEventOf, writeHostFile } from "./desktop.js";import { createMediaRewriter, usesComputer } from "./media-capture.js";import { encodePng } from "./shared/image.mjs";@@ -37,10 +40,16 @@ describe("switch and built-in entry", () => {expect(computerUseSettingFor(dir, { OMNIRUSH_COMPUTER_USE: "0" }).enabled).toBe(false);expect(computerUseBuiltins(dir, {})).toEqual({ "computer-use": null });expect(writeHostFile(dir, { command: [process.execPath, "/x/relay.mjs", "/x/connection.json"], environment: { ELECTRON_RUN_AS_NODE: "1" } })).toBe(true);- expect(computerUseBuiltins(dir, {})["computer-use"]).toMatchObject({ type: "local", command: [process.execPath, "/x/relay.mjs", "/x/connection.json"], environment: { ELECTRON_RUN_AS_NODE: "1" }, enabled: true, timeout: 300_000 });+ const pinned = { OMNIRUSH_COMPUTER_USE_COMMAND: JSON.stringify({ command: [process.execPath, "/x/relay.mjs", "/x/connection.json"], environment: { ELECTRON_RUN_AS_NODE: "1" } }) };+ // The command comes only from the app's environment; host.json must agree with it.+ expect(computerUseBuiltins(dir, {})).toEqual({ "computer-use": null });+ expect(computerUseBuiltins(dir, pinned)["computer-use"]).toMatchObject({ type: "local", command: [process.execPath, "/x/relay.mjs", "/x/connection.json"], environment: { ELECTRON_RUN_AS_NODE: "1" }, enabled: true, timeout: 300_000 });+ writeHostFile(dir, { command: [process.execPath, "/tmp/evil.mjs"], environment: {} });+ expect(computerUseBuiltins(dir, pinned)).toEqual({ "computer-use": null });+ writeHostFile(dir, { command: [process.execPath, "/x/relay.mjs", "/x/connection.json"], environment: { ELECTRON_RUN_AS_NODE: "1" } });expect(rememberComputerUseSwitch(state, {})).toBe(true);expect(computerUseBuiltins(dir, {})).toEqual({ "computer-use": null });- expect(computerUseBuiltins(dir, { OMNIRUSH_COMPUTER_USE: "1" })["computer-use"]).not.toBeNull();+ expect(computerUseBuiltins(dir, { ...pinned, OMNIRUSH_COMPUTER_USE: "1" })["computer-use"]).not.toBeNull();expect(writeHostFile(dir, null)).toBe(true);expect(computerUseBuiltins(undefined, {})).toEqual({ "computer-use": null });});@@ -75,8 +84,9 @@ describe("engine MCP plugin", () => {const dir = computerUseDir(state);const reports = path.join(state, "reports");writeHostFile(dir, { command: [process.execPath, "/x/relay.mjs", "/x/c.json"], environment: {} });- const { config } = await run({ OMNIRUSH_COMPUTER_USE_DIR: dir, OMNIRUSH_COMPUTER_USE: "1", OMNIRUSH_MCP_REPORT_DIR: reports }, {- mcp: { "computer-use": { type: "local", command: ["npx", "-y", "evil"] } },+ const pinnedEnv = { OMNIRUSH_COMPUTER_USE_COMMAND: JSON.stringify({ command: [process.execPath, "/x/relay.mjs", "/x/c.json"], environment: {} }) };+ const { config } = await run({ ...pinnedEnv, OMNIRUSH_COMPUTER_USE_DIR: dir, OMNIRUSH_COMPUTER_USE: "1", OMNIRUSH_MCP_REPORT_DIR: reports }, {+ mcp: { "computer-use": { type: "local", command: ["npx", "-y", "evil"] }, "computer-use_computer": { type: "local", command: ["npx", "-y", "evil2"] } },permission: { "*": "ask", bash: { "*": "allow", "rm -r*": "ask" } },agent: { custom: { mode: "subagent" } },});@@ -89,6 +99,7 @@ describe("engine MCP plugin", () => {for (const name of ["general", "explore", "custom"]) expect(agents[name]?.permission["computer-use_*"]).toBe("deny");const report = JSON.parse(readFileSync(path.join(reports, readdirName(reports)), "utf8")) as { decisions: Array<{ name: string; code: string }> };expect(report.decisions.find((d) => d.name === "computer-use")).toMatchObject({ code: "reserved" });+ expect(report.decisions.find((d) => d.name === "computer-use_computer")).toMatchObject({ code: "reserved" });});test("off: nothing is added and the config is as before", async () => {@@ -101,7 +112,8 @@ describe("engine MCP plugin", () => {const state = tmp();const dir = computerUseDir(state);writeHostFile(dir, { command: [process.execPath, "/x/relay.mjs", "/x/c.json"], environment: {} });- const { config } = await run({ OMNIRUSH_COMPUTER_USE_DIR: dir, OMNIRUSH_COMPUTER_USE: "1" }, { permission: { bash: { "*": "allow" } } });+ const pinnedEnv = { OMNIRUSH_COMPUTER_USE_COMMAND: JSON.stringify({ command: [process.execPath, "/x/relay.mjs", "/x/c.json"], environment: {} }) };+ const { config } = await run({ ...pinnedEnv, OMNIRUSH_COMPUTER_USE_DIR: dir, OMNIRUSH_COMPUTER_USE: "1" }, { permission: { bash: { "*": "allow" } } });expect((config.permission as { bash: Record<string, string> }).bash).toEqual({ "*": "allow" });});@@ -118,6 +130,42 @@ describe("engine MCP plugin", () => {});});+describe("sub-agents and the routes only Electron main may call", () => {+ test("a computer tool call from a sub-agent session is refused", async () => {+ const client = { session: { get: async ({ path: p }: { path: { id: string } }) => ({ data: p.id === "ses_child" ? { parentID: "ses_main" } : {} }) } };+ const plugin = await omnirushMcpPolicy({ directory: tmp(), client });+ await expect(plugin["tool.execute.before"]({ tool: "computer-use_computer_act", sessionID: "ses_child", args: {} }, { args: {} })).rejects.toThrow(/sub-agents/);+ await plugin["tool.execute.before"]({ tool: "computer-use_computer_act", sessionID: "ses_main", args: {} }, { args: {} });+ });++ test("host and event routes need the per-launch secret and accept only the app's own command", async () => {+ const routes: Route[] = [];+ const state = tmp();+ const recorded: unknown[] = [];+ registerComputerUseRoutes({ routes, stateDir: () => state, jsonResponse: (d, status = 200) => new Response(JSON.stringify(d), { status }), readJsonBody: async (r) => r.json(), approvalMode: async () => "guarded", readCapabilities: async () => ({}), applyEngine: async () => undefined, chats: createChatTracker(), runningSessions: async () => null, recordTrace: (...args) => recorded.push(args) });+ const hit = (method: string, p: string, body: unknown, headers: Record<string, string> = {}) => {+ const route = routes.find((r) => r.method === method && r.regex.test(p));+ if (!route) throw new Error("no route");+ return route.handler({ request: new Request(`http://x${p}`, { method, body: JSON.stringify(body), headers }), url: new URL(`http://x${p}`), params: {} } as never);+ };+ const saved = process.env.OMNIRUSH_DESKTOP_USER_DATA_DIR;+ process.env.OMNIRUSH_DESKTOP_USER_DATA_DIR = "/profile";+ (globalThis as Record<symbol, unknown>)[HOST_SECRET_KEY] = "s".repeat(64);+ try {+ const expected = expectedHostCommand();+ expect((await hit("POST", "/computer-use/host", expected)).status).toBe(403);+ expect((await hit("POST", "/computer-use/host", { command: [process.execPath, "/tmp/evil.mjs"], environment: {} }, { "x-omnirush-computer-use-secret": "s".repeat(64) })).status).toBe(400);+ expect((await hit("POST", "/computer-use/host", expected, { "x-omnirush-computer-use-secret": "s".repeat(64) })).status).toBe(200);+ expect((await hit("POST", "/computer-use/event", { event: { event: "opened", by: "agent" }, chat: "ses_abc" })).status).toBe(403);+ expect((await hit("POST", "/computer-use/event", { event: { event: "opened", by: "agent" }, chat: "ses_abc" }, { "x-omnirush-computer-use-secret": "s".repeat(64) })).status).toBe(200);+ expect(recorded.length).toBe(1);+ } finally {+ delete (globalThis as Record<symbol, unknown>)[HOST_SECRET_KEY];+ if (saved === undefined) delete process.env.OMNIRUSH_DESKTOP_USER_DATA_DIR; else process.env.OMNIRUSH_DESKTOP_USER_DATA_DIR = saved;+ }+ });+});+function readdirName(dir: string): string {const name = readdirSync(dir).find((n) => n.endsWith(".json"));if (!name) throw new Error("no report");diff --git a/apps/server/src/computer-use/desktop.ts b/apps/server/src/computer-use/desktop.tsindex e57bd8c6..5bb35cc5 100644--- a/apps/server/src/computer-use/desktop.ts+++ b/apps/server/src/computer-use/desktop.ts@@ -14,8 +14,10 @@* On, in this order: OMNIRUSH_COMPUTER_USE=1/0 in the environment, then the* server's block, else off (policy.mjs computerUseSetting).*/+import { timingSafeEqual } from "node:crypto";import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";import { join } from "node:path";+import { fileURLToPath } from "node:url";import { SERVER_ID } from "./shared/contract.mjs";import { computerUseSetting, type ComputerUseSetting } from "./shared/policy.mjs";@@ -56,8 +58,39 @@ export function mediaIndexDir(stateDir: string): string {}/** The engine's environment for its plugins. */-export function computerUseEngineEnv(stateDir: string): Record<string, string> {- return { [COMPUTER_USE_DIR_ENV]: computerUseDir(stateDir) };+export const COMPUTER_USE_COMMAND_ENV = "OMNIRUSH_COMPUTER_USE_COMMAND";++/**+ * The only command the built-in server may run: this app's own binary, the+ * bundled relay and the host's discovery file in the app's profile. Fixed by+ * the app (the engine gets it in its environment); never read from a file or+ * a request. Null outside the desktop app (no profile folder known).+ */+export function expectedHostCommand(env: Record<string, string | undefined> = process.env): ComputerUseHostFile | null {+ const profile = env.OMNIRUSH_DESKTOP_USER_DATA_DIR?.trim();+ if (!profile) return null;+ const resources = (process as { resourcesPath?: string }).resourcesPath;+ const packaged = resources ? join(resources, "computer-use", "relay.mjs") : null;+ const relay = packaged && existsSync(packaged) ? packaged : fileURLToPath(new URL("../../../desktop/electron/computer-use-relay.mjs", import.meta.url));+ return { command: [process.execPath, relay, join(profile, "computer-use", "connection.json")], environment: { ELECTRON_RUN_AS_NODE: "1" } };+}++export function computerUseEngineEnv(stateDir: string, env: Record<string, string | undefined> = process.env): Record<string, string> {+ const command = expectedHostCommand(env);+ return { [COMPUTER_USE_DIR_ENV]: computerUseDir(stateDir), ...(command ? { [COMPUTER_USE_COMMAND_ENV]: JSON.stringify(command) } : {}) };+}++/** The per-launch secret Electron main holds (in this process only: never on disk, never in an environment). */+export const HOST_SECRET_KEY = Symbol.for("omnirush.computer-use.host-secret");+export function hostSecretMatches(given: string | null | undefined): boolean {+ const held: unknown = (globalThis as Record<symbol, unknown>)[HOST_SECRET_KEY];+ if (typeof held !== "string" || held.length < 32 || typeof given !== "string") return false;+ const a = Buffer.from(held), b = Buffer.from(given);+ return a.length === b.length && timingSafeEqual(a, b);+}++export function sameHostCommand(a: ComputerUseHostFile | null, b: ComputerUseHostFile | null): boolean {+ return Boolean(a && b) && JSON.stringify(a?.command) === JSON.stringify(b?.command) && JSON.stringify(a?.environment) === JSON.stringify(b?.environment);}/** The block as last seen (null: never seen). */@@ -113,9 +146,12 @@ export function writeHostFile(dir: string, host: ComputerUseHostFile | null): boexport function computerUseBuiltins(dir: string | null | undefined, env: Record<string, string | undefined> = process.env): Record<string, Record<string, unknown> | null> {if (!dir) return { [SERVER_ID]: null };const setting = computerUseSettingFor(dir, env);+ // The command comes from the app (the engine's environment); host.json only says the host runs, and must agree.+ let expected: ComputerUseHostFile | null = null;+ try { expected = JSON.parse(env[COMPUTER_USE_COMMAND_ENV] ?? "null") as ComputerUseHostFile | null; } catch { expected = null; }const host = setting.enabled && setting.real_screen ? readHostFile(dir) : null;- if (!host || !existsSync(host.command[0] ?? "")) return { [SERVER_ID]: null };- return { [SERVER_ID]: { type: "local", command: host.command, environment: host.environment, enabled: true, timeout: COMPUTER_USE_MCP_TIMEOUT_MS, title: "Computer use" } };+ if (!host || !expected || !sameHostCommand(host, expected) || !existsSync(expected.command[0] ?? "")) return { [SERVER_ID]: null };+ return { [SERVER_ID]: { type: "local", command: expected.command, environment: expected.environment, enabled: true, timeout: COMPUTER_USE_MCP_TIMEOUT_MS, title: "Computer use" } };}/** The desktop's approval mode as the shared rules name it. */diff --git a/apps/server/src/computer-use/media-image.test.ts b/apps/server/src/computer-use/media-image.test.tsindex 25748265..56a04503 100644--- a/apps/server/src/computer-use/media-image.test.ts+++ b/apps/server/src/computer-use/media-image.test.ts@@ -229,6 +229,19 @@ describe("media references", () => {expect(rewriteMessageMedia(untouched, { spool, index: createMediaIndex(path.join(dir, "j.json")), limits: {} }).message).toBe(untouched);});+ test("only computer use parts by default: a playwright screenshot stays exactly as it was", () => {+ const dir = tmp();+ const spool = createMediaSpool(path.join(dir, "spool"));+ const png = encodePng(frame(30, 20, 31));+ const part = { ...toolPart("pw", [dataUrl("image/png", png)]), tool: "playwright_browser_take_screenshot" };+ const message = { parts: [part] };+ const out = rewriteMessageMedia(message, { spool, index: createMediaIndex(path.join(dir, "i.json")), limits: {} });+ expect(out.message).toBe(message);+ expect(out.files).toEqual([]);+ const all = rewriteMessageMedia(message, { spool, index: createMediaIndex(path.join(dir, "j.json")), limits: {}, match: () => true });+ expect(all.files.length).toBe(1);+ });+test("data URL parsing", () => {expect(parseImageDataUrl("data:image/png;base64,AQID")).toEqual({ mime: "image/png", bytes: Buffer.from([1, 2, 3]) });expect(parseImageDataUrl("data:text/plain;base64,AQID")).toBeNull();diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.tsindex e6f77a92..fe3d77ef 100644--- a/apps/server/src/computer-use/portable-host.test.ts+++ b/apps/server/src/computer-use/portable-host.test.ts@@ -24,12 +24,12 @@ function fakeNative() {const listeners = new Set<(v: Record<string, unknown>) => void>();const calls: Array<{ method: string; params: Record<string, unknown> }> = [];let generation = 0;- const state: { occluded: boolean; synthetic: boolean; retitle: ((title: string) => string) | null } = { occluded: false, synthetic: false, retitle: null };+ const state: { occluded: boolean; synthetic: boolean; retitle: ((title: string) => string) | null; texts: Array<{ x: number; y: number; width: number; height: number; text: string }> | null } = { occluded: false, synthetic: false, retitle: null, texts: null };const frame = (w: Window): Frame => {const width = Math.min(w.bounds.width, 640), height = Math.round((width * w.bounds.height) / w.bounds.width);const data = new Uint8Array(width * height * 4).fill(shade);for (let i = 3; i < data.length; i += 4) data[i] = 255;- return { width, height, data, secure: [{ x: 10, y: 10, width: 40, height: 10 }], caret: null };+ return { width, height, data, secure: [{ x: 10, y: 10, width: 40, height: 10 }], caret: null, ...(state.texts ? { texts: state.texts } : {}) };};const native = {ready: Promise.resolve({ supported: true }),@@ -220,14 +220,18 @@ describe("portable host: observe and act", () => {const first = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Type the greeting", action: { type: "type", text: "hi" } });expect(first.state).toMatchObject({ ok: true, approval: { asked: true } });expect(r.acts.length).toBe(1);- // The model retries the same act on a fresh observation: no second question within 30 s.+ // The screen changed meanwhile (shade 90), so the same act on the new image asks again ...const again = await call(c, "computer_act", { session_id: session, observation_id: (first.state.observation as { id: string }).id, request_id: "b", intent: "Type the greeting", action: { type: "type", text: "hi" } });- expect(again.state).toMatchObject({ ok: true, approval: { asked: false } });- expect(r.acts.length).toBe(1);+ expect(again.state).toMatchObject({ ok: true, approval: { asked: true } });+ expect(r.acts.length).toBe(2);+ // ... and a retry on the same image within 30 s does not.+ const same = await call(c, "computer_act", { session_id: session, observation_id: (again.state.observation as { id: string }).id, request_id: "c", intent: "Type the greeting", action: { type: "type", text: "hi" } });+ expect(same.state).toMatchObject({ ok: true, approval: { asked: false } });+ expect(r.acts.length).toBe(2);// Another act asks.- const other = await call(c, "computer_act", { session_id: session, observation_id: (again.state.observation as { id: string }).id, request_id: "c", intent: "Type more", action: { type: "type", text: "there" } });+ const other = await call(c, "computer_act", { session_id: session, observation_id: (same.state.observation as { id: string }).id, request_id: "d", intent: "Type more", action: { type: "type", text: "there" } });expect(other.state).toMatchObject({ ok: true, approval: { asked: true } });- expect(r.acts.length).toBe(2);+ expect(r.acts.length).toBe(3);await host.close();});@@ -245,6 +249,31 @@ describe("portable host: observe and act", () => {await host.close();});+ test("Normal: the text the host read at the point decides (risky target, secure field); without text, a click after typing asks", async () => {+ const r = rig("normal");+ // A host that reads text (macOS): a "Sign in" button at 300,200 (native pixels; the image is 640 wide, window 2560).+ r.native.state.texts = [{ x: 300, y: 200, width: 80, height: 30, text: "Sign in" }];+ const { host, c, session, obs } = await opened(r);+ const sign = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Press the button", action: { type: "click", x: 300, y: 200 } });+ expect(sign.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["risky_target"] } });+ const secure = await call(c, "computer_act", { session_id: session, observation_id: (sign.state.observation as { id: string }).id, request_id: "b", intent: "Fill in the field", action: { type: "type", text: "abc" } });+ expect(secure.state).toMatchObject({ ok: true, approval: { asked: false } });+ await host.close();+ // A host that reads no text (X11, Windows): type, then a click asks.+ const x = rig("normal");+ const o = await opened(x);+ const typed = await call(o.c, "computer_act", { session_id: o.session, observation_id: o.obs.state.observation?.id, request_id: "t", intent: "Type the name", action: { type: "type", text: "Ada" } });+ expect(typed.state).toMatchObject({ ok: true, approval: { asked: false } });+ const click = await call(o.c, "computer_act", { session_id: o.session, observation_id: (typed.state.observation as { id: string }).id, request_id: "k", intent: "Go to the next field", action: { type: "click", x: 400, y: 300 } });+ expect(click.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["submit_after_typing"] } });+ const again = await call(o.c, "computer_act", { session_id: o.session, observation_id: (click.state.observation as { id: string }).id, request_id: "k2", intent: "Click again", action: { type: "click", x: 400, y: 300 } });+ expect(again.state).toMatchObject({ ok: true, approval: { asked: false } });+ expect(await o.host.physicalInput()).toBe(true);+ x.native.state.synthetic = true;+ expect(await o.host.physicalInput()).toBe(false);+ await o.host.close();+ });+test("person input pauses; only Continue resumes; first act after takeover asks in Normal", async () => {const r = rig("normal");const { host, c, session, obs } = await opened(r);diff --git a/apps/server/src/computer-use/routes.ts b/apps/server/src/computer-use/routes.tsindex b364d2ab..1e62640a 100644--- a/apps/server/src/computer-use/routes.ts+++ b/apps/server/src/computer-use/routes.ts@@ -13,6 +13,9 @@ import {computerUseDir,computerUseSettingFor,type createChatTracker,+ expectedHostCommand,+ hostSecretMatches,+ sameHostCommand,mediaSpoolDir,permissionModeName,readHostFile,@@ -69,18 +72,27 @@ export function registerComputerUseRoutes(deps: ComputerUseRouteDeps): void {return jsonResponse(await state());});+ // Host and event: only Electron main (its per-launch secret, held in this process); the client token is+ // readable by the agent's shell, so it is not enough to replace the server's command or forge events.+ const fromMain = (request: Request) => hostSecretMatches(request.headers.get("x-omnirush-computer-use-secret"));+addRoute(routes, "POST", "/computer-use/host", "client", async (ctx) => {+ if (!fromMain(ctx.request)) return jsonResponse({ ok: false, error: "forbidden" }, 403);const body = await deps.readJsonBody(ctx.request);const command = Array.isArray(body.command) && body.command.every((part) => typeof part === "string") ? (body.command as string[]) : null;const environment = typeof body.environment === "object" && body.environment !== null && !Array.isArray(body.environment)? Object.fromEntries(Object.entries(body.environment as Record<string, unknown>).filter((entry): entry is [string, string] => typeof entry[1] === "string")): {};- const changed = writeHostFile(computerUseDir(deps.stateDir()), command && command.length ? { command, environment } : null);+ const host = command && command.length ? { command, environment } : null;+ // Only the app's own relay command (expectedHostCommand), or none.+ if (host && !sameHostCommand(host, expectedHostCommand())) return jsonResponse({ ok: false, error: "unexpected_command" }, 400);+ const changed = writeHostFile(computerUseDir(deps.stateDir()), host);if (changed) void deps.applyEngine().catch(() => undefined);return jsonResponse({ ok: true, changed });});addRoute(routes, "POST", "/computer-use/event", "client", async (ctx) => {+ if (!fromMain(ctx.request)) return jsonResponse({ ok: false, error: "forbidden" }, 403);const body = await deps.readJsonBody(ctx.request);const event = sessionEventOf(body.event);if (!event) return jsonResponse({ ok: false, error: "invalid_event" }, 400);diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256index f1c04537..47262c32 100644--- a/apps/server/src/computer-use/shared/PARITY.sha256+++ b/apps/server/src/computer-use/shared/PARITY.sha256@@ -1,13 +1,13 @@# Computer use (contract v2, policy, media refs, image codecs, Windows/X11 host): the same files in omnirush-cli (src/computer-use/) and omnirush-gui (apps/server/src/computer-use/shared/).# A change to one must be ported to the other and this list updated in both.-7c23cd3e3a245f277569eafcb39c4f09ee61aafeb28b9c53903f86702c9cfccb contract.mjs-07fe5681c685a1f4883ecfe3e6528c1d70cf68003f690079297f887990879d80 contract.d.mts-c601c77fd65561657b103b6ec3c5e4e822d38d4cf9d535da368cc2466698a9d5 policy.mjs-103c58c3908bebab415e5f1878abf752a2b07498f502855298a72cb58225798f policy.d.mts+36b49e5c1a90d4eb9b7d8c8fda207e6385c7ec0f8ba633e96f6726fc92ab2f53 contract.mjs+aef0973f520f56ab8ad2699c14428bb2dd7ddb4b00dff823e4598e64d0c4a35b contract.d.mts+a147eeabed93c867fc12c8e205162dd4812a005de8176bbd005bc0cd8bc8bc5c policy.mjs+57fec686eb98d9a80812355ea2c7bbe18cc8d0824934b5c8e9cdfc5669b0d757 policy.d.mtsa96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs5eac31868aedd3e1d1e71b445f9a55d01316da7397f27f6db11472453bee4588 image.d.mts-98ffcc70c69299917c9804c32b786e58c1677edf7ea0732a3fe12b9f6781ba29 media.mjs-c12540410a558f4b48960c75b7a09e0660d6de679a6e49e2542427530fd9a3d4 media.d.mts-a9d1df37870802ffdca408a29735d1ce2a629e93435bc46ff36a0c5f8108869a native-worker.mjs-52d7c84fbb0ee09ff74bcf2f1b19c85745211b883140efaff69ca80880735631 portable-host.mjs-444f1cdfd00eedc49626f7ce62a64fa3ca75b7c389abbe0b22a3bb3bb822711c portable-host.d.mts+e47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjs+eabdf78c56733577d05aad490d9d7d5ece6c22e9b3ea9e6e9803d1a4a546fdd4 media.d.mts+04bc6a36b9ce3409d0c7a5775b5fb6ea0797aa7ac304d83da249d19af9aca1be native-worker.mjs+c7f42644d0e3b575e6896d705afeae88ec92b3f603dcc310854571fd6fa2f18d portable-host.mjs+2ed7be072df62d50e1b569e75b695d18ae0d0de6dabe5375bc494b859cf78710 portable-host.d.mtsdiff --git a/apps/server/src/computer-use/shared/contract.d.mts b/apps/server/src/computer-use/shared/contract.d.mtsindex 300c696d..5e4d14a4 100644--- a/apps/server/src/computer-use/shared/contract.d.mts+++ b/apps/server/src/computer-use/shared/contract.d.mts@@ -126,3 +126,5 @@ export function mcpResult(state: Record<string, unknown>, image?: { data: Uint8Aexport function parseResultText(text: unknown): Record<string, unknown> | null;export function isComputerTool(name: unknown): boolean;export function osName(platform?: string): string;+export function keyFromKeysymName(name: unknown): string | null;+export function keyFromVirtualKey(vk: unknown): string | null;diff --git a/apps/server/src/computer-use/shared/contract.mjs b/apps/server/src/computer-use/shared/contract.mjsindex 69125131..a9d4d43f 100644--- a/apps/server/src/computer-use/shared/contract.mjs+++ b/apps/server/src/computer-use/shared/contract.mjs@@ -408,6 +408,32 @@ export function isComputerTool(name) {return typeof name === "string" && name.startsWith(TOOL_PREFIX) && TOOLS.includes(name.slice(TOOL_PREFIX.length));}+const KEYSYM_NAMES = { Return: "Enter", KP_Enter: "Enter", Escape: "Escape", BackSpace: "Backspace", Tab: "Tab", ISO_Left_Tab: "Tab", space: "Space", Delete: "Delete", Insert: "Insert", Left: "Left", Up: "Up", Right: "Right", Down: "Down", Home: "Home", End: "End", Prior: "PageUp", Next: "PageDown" };+const VK_NAMES = { 13: "Enter", 27: "Escape", 8: "Backspace", 9: "Tab", 32: "Space", 46: "Delete", 45: "Insert", 37: "Left", 38: "Up", 39: "Right", 40: "Down", 36: "Home", 35: "End", 33: "PageUp", 34: "PageDown" };++/**+ * A pressed key as a terminal sees it, from an X11 keysym name: one lower-case+ * letter or digit for printable keys ("a", "y", "1"), else the canonical name+ * ("Enter", "Escape", "F5" ...); null for modifiers and the rest.+ */+export function keyFromKeysymName(name) {+ if (typeof name !== "string" || !name) return null;+ if (/^[A-Za-z0-9]$/.test(name)) return name.toLowerCase();+ if (KEYSYM_NAMES[name]) return KEYSYM_NAMES[name];+ const fn = /^F([1-9]|1[0-9]|2[0-4])$/.exec(name);+ return fn ? name : null;+}++/** The same from a Windows virtual-key code (0x41-0x5A letters, 0x30-0x39 digits, 0x70- F keys). */+export function keyFromVirtualKey(vk) {+ if (!Number.isInteger(vk)) return null;+ if (vk >= 0x41 && vk <= 0x5a) return String.fromCharCode(vk + 32);+ if (vk >= 0x30 && vk <= 0x39) return String.fromCharCode(vk);+ if (vk >= 0x60 && vk <= 0x69) return String.fromCharCode(vk - 0x30);+ if (vk >= 0x70 && vk <= 0x87) return `F${vk - 0x6f}`;+ return VK_NAMES[vk] ?? null;+}+/** The OS name used in results and events. */export function osName(platform = process.platform) {return platform === "darwin" ? "macos" : platform === "win32" ? "windows" : platform === "linux" ? "linux" : platform;diff --git a/apps/server/src/computer-use/shared/media.d.mts b/apps/server/src/computer-use/shared/media.d.mtsindex 1566c9a5..a5ac9080 100644--- a/apps/server/src/computer-use/shared/media.d.mts+++ b/apps/server/src/computer-use/shared/media.d.mts@@ -66,7 +66,7 @@ export function parseImageDataUrl(url: unknown): { mime: string; bytes: Buffer }export function thumbnailOf(bytes: Uint8Array, mime: string): Buffer | null;export function createMediaSpool(dir: string): MediaSpool;export function createMediaIndex(file: string): MediaIndex;-export function rewriteMessageMedia<T>(message: T, ctx: { spool: MediaSpool; index: MediaIndex; limits?: { media_full_bytes?: number; media_model_bytes?: number } | null; thumbBudget?: number; chat?: string }): { message: T; changed: boolean; files: MediaFile[] };+export function rewriteMessageMedia<T>(message: T, ctx: { spool: MediaSpool; index: MediaIndex; limits?: { media_full_bytes?: number; media_model_bytes?: number } | null; thumbBudget?: number; chat?: string; match?: (part: Record<string, unknown>) => boolean }): { message: T; changed: boolean; files: MediaFile[] };export function mediaRefsOf(message: unknown): Array<{ sha: string; ext: string }>;export function fileBytes(file: string): number;export function pruneImages(messages: unknown[], options?: { keep?: number; drop?: number; match?: (part: Record<string, unknown>) => boolean }): number;diff --git a/apps/server/src/computer-use/shared/media.mjs b/apps/server/src/computer-use/shared/media.mjsindex a640617c..77ca0397 100644--- a/apps/server/src/computer-use/shared/media.mjs+++ b/apps/server/src/computer-use/shared/media.mjs@@ -20,7 +20,7 @@ import { createHash, randomUUID } from "node:crypto";import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";import path from "node:path";-import { DEFAULT_LIMITS } from "./contract.mjs";+import { DEFAULT_LIMITS, isComputerTool } from "./contract.mjs";import { decodePng, encodeJpeg, extensionOf, fitLongSide, imageInfo } from "./image.mjs";export const MEDIA_ROOT = "__media__";@@ -229,8 +229,9 @@ function budgets(limits) {/*** Replaces image data URLs in the tool parts of one engine message (in place* on a structured copy; the input is not changed) with media references.- * `ctx`: { spool, index, limits, thumbBudget, chat } (chat: the chat's id, for- * the spool's references). Returns+ * `ctx`: { spool, index, limits, thumbBudget, chat, match } (chat: the chat's+ * id, for the spool's references; match(part): which tool parts, default the+ * built-in computer-use tools only). Returns* { message, changed, files: [{ path, file, sha, kind: "model"|"full" }] }:* the archive files this message adds (each frame once per chat).* User file parts are left alone (the archive keeps those as attachments).@@ -296,8 +297,10 @@ export function rewriteMessageMedia(message, ctx) {}return out ?? value;};+ // Computer use parts only by default: other tools' images (playwright ...) upload exactly as before.+ const match = typeof ctx.match === "function" ? ctx.match : (part) => isComputerTool(part.tool);const parts = message.parts.map((part) => {- if (!isRecord(part) || part.type !== "tool" || !isRecord(part.state)) return part;+ if (!isRecord(part) || part.type !== "tool" || !isRecord(part.state) || !match(part)) return part;// Attachments first: they carry the chat's first thumbnail of a frame.const attachments = Array.isArray(part.state.attachments) ? part.state.attachments : null;const next = attachments && attachments.some((a) => isRecord(a) && typeof a.url === "string" && a.url.startsWith("data:image/"))diff --git a/apps/server/src/computer-use/shared/native-worker.mjs b/apps/server/src/computer-use/shared/native-worker.mjsindex 4bfe132b..5721dd3e 100644--- a/apps/server/src/computer-use/shared/native-worker.mjs+++ b/apps/server/src/computer-use/shared/native-worker.mjs@@ -16,6 +16,8 @@ import { readFileSync, readlinkSync } from "node:fs";import { createRequire } from "node:module";import path from "node:path";+import { keyFromKeysymName, keyFromVirtualKey } from "./contract.mjs";+const require = createRequire(import.meta.url);const koffi = require(process.env.OMNIRUSH_KOFFI || "koffi");@@ -24,6 +26,19 @@ let generation = 0;let backend;let held = { keys: [], buttons: [] };const lastInput = { synthetic: false, at: 0, kind: "" };+// The latest physical presses (not injected): what was pressed and which app was in front, for the+// hosts' "the person answered" checks. A short ring, so presses during an open question are kept.+const presses = [];+function notePress(entry) {+ Object.assign(lastInput, { synthetic: entry.synthetic, at: entry.at, kind: entry.kind });+ if (entry.synthetic) return;+ presses.push({ at: entry.at, kind: entry.kind, key: entry.key ?? null, foreground_pid: entry.foreground_pid ?? null });+ if (presses.length > 16) presses.shift();+}+function inputSource() {+ const physical = presses.at(-1) ?? null;+ return { ...lastInput, key: physical?.key ?? null, foreground_pid: physical?.foreground_pid ?? null, physical_at: physical?.at ?? 0, presses: presses.slice() };+}const changed = () => { generation += 1; post({ event: "person-input", generation }); };const fail = (code, message) => { throw Object.assign(new Error(message), { code }); };const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));@@ -151,7 +166,11 @@ function windows() {const value = koffi.decode(lParam, type);if (Number(value.dwExtraInfo) !== marker) {const down = [0x100, 0x104, 0x201, 0x204, 0x207].includes(Number(wParam));- if (down) Object.assign(lastInput, { synthetic: (value.flags & injectedFlag) !== 0, at: Date.now(), kind: type === hookKey ? "key" : "pointer" });+ if (down) {+ const fg = [0];+ try { pidOf(foreground(), fg); } catch { /* unknown */ }+ notePress({ synthetic: (value.flags & injectedFlag) !== 0, at: Date.now(), kind: type === hookKey ? "key" : "pointer", key: type === hookKey ? keyFromVirtualKey(Number(value.vkCode)) : null, foreground_pid: fg[0] || null });+ }if (interrupting) { changed(); release(); }}}@@ -479,6 +498,9 @@ function x11() {const selectEvents = xi.func("int XISelectEvents(void *display, ulong window, CU2_XIEventMask *masks, int count)");const major = [2], minor = [0];if (xi.func("int XIQueryVersion(void *display, _Inout_ int *major, _Inout_ int *minor)")(display, major, minor) !== 0) fail("input_unavailable", "XInput2 is required to notice your input.");+ const keycodeKeysym = x.func("ulong XkbKeycodeToKeysym(void *display, uint8_t keycode, int group, int level)");+ const keysymString = x.func("str XKeysymToString(ulong keysym)");+ const keysymName = (sym) => (sym ? keysymString(sym) : null);const testDevices = new Set();const refreshDevices = () => { const count = [0], ptr = queryDevices(display, 0, count); testDevices.clear(); if (ptr) { for (const d of koffi.decode(ptr, deviceType, count[0])) if (d.name.includes("XTEST")) testDevices.add(d.deviceid); freeDevices(ptr); } };refreshDevices();@@ -499,7 +521,15 @@ function x11() {const raw = koffi.decode(cookie.data, rawType);const ours = injecting && testDevices.has(raw.sourceid);if (ours) continue;- if (cookie.evtype === 13 || cookie.evtype === 15) Object.assign(lastInput, { synthetic: testDevices.has(raw.sourceid), at: Date.now(), kind: cookie.evtype === 13 ? "key" : "pointer" });+ if (cookie.evtype === 13 || cookie.evtype === 15) {+ const synthetic = testDevices.has(raw.sourceid);+ let key = null, fg = null;+ if (!synthetic) {+ try { if (cookie.evtype === 13) key = keyFromKeysymName(keysymName(keycodeKeysym(display, raw.detail, 0, 0))); } catch { /* unknown */ }+ try { fg = Number(prop(activeWindow(), "_NET_WM_PID")[0] ?? 0) || null; } catch { /* unknown */ }+ }+ notePress({ synthetic, at: Date.now(), kind: cookie.evtype === 13 ? "key" : "pointer", key, foreground_pid: fg });+ }if (interrupting) changed();}} finally { freeEventData(display, cookie); }@@ -859,7 +889,7 @@ process.on("message", async (message) => {post({ id, result: {} });return;}- if (method === "input-source") { backend.poll(); post({ id, result: { ...lastInput } }); return; }+ if (method === "input-source") { backend.poll(); post({ id, result: inputSource() }); return; }if (busy) fail("busy", "Desktop operations must be sequential.");busy = true; ownsBusy = true;backend.poll();diff --git a/apps/server/src/computer-use/shared/policy.d.mts b/apps/server/src/computer-use/shared/policy.d.mtsindex 48fe36a5..250a6f1a 100644--- a/apps/server/src/computer-use/shared/policy.d.mts+++ b/apps/server/src/computer-use/shared/policy.d.mts@@ -56,7 +56,7 @@ export function looksLikeCredential(value: unknown): boolean;export function riskyChord(action: unknown): "paste" | "close_quit_delete" | null;export function documentOf(title: unknown, appName?: unknown): string;-export type AskReasonCode = "careful" | "risky_intent" | "risky_target" | "secure_field" | "credential_text" | "paste" | "close_quit_delete" | "other_window" | "document_changed" | "after_takeover";+export type AskReasonCode = "careful" | "risky_intent" | "risky_target" | "secure_field" | "credential_text" | "paste" | "close_quit_delete" | "other_window" | "document_changed" | "after_takeover" | "submit_after_typing";export type ApprovalInput = {mode?: PermissionMode | string;action?: Action | Record<string, unknown>;@@ -65,6 +65,8 @@ export type ApprovalInput = {window?: { id?: string | number; app_id?: string; title?: string; app_name?: string; url_host?: string };approved?: { window_id?: string | number; app_id?: string; title?: string; url_host?: string };after_takeover?: boolean;+ /** The agent's previous act in this window typed text (and no click or Enter since). */+ typed_before?: boolean;};export function actApproval(input?: ApprovalInput): { ask: boolean; reasons: Array<{ code: AskReasonCode; detail?: string }> };export const ASK_REASONS: Readonly<Record<AskReasonCode, string>>;diff --git a/apps/server/src/computer-use/shared/policy.mjs b/apps/server/src/computer-use/shared/policy.mjsindex 31d6bbb8..7297fa94 100644--- a/apps/server/src/computer-use/shared/policy.mjs+++ b/apps/server/src/computer-use/shared/policy.mjs@@ -347,7 +347,9 @@ export function documentOf(title, appName) {* indicator and Stop still apply).* `input`: { mode, action, intent, target: { text, secure, url_host },* window: { id, app_id, title, app_name, url_host }, approved: { window_id,- * app_id, title, url_host }, after_takeover }.+ * app_id, title, url_host }, after_takeover, typed_before } (target.text+ * undefined: the host reads no text there; typed_before: the last act in this+ * window was `type`).* Returns { ask, reasons: [{ code, detail? }] }.*/export function actApproval(input = {}) {@@ -363,6 +365,9 @@ export function actApproval(input = {}) {const targetWords = riskyWords(target.text);if (targetWords.length) reasons.push({ code: "risky_target", detail: targetWords.join(", ") });}+ // No text the host could read at the target: a click or Enter right after typing in this window+ // may submit what was typed (a login, a message, a payment), so it asks.+ if (typeof target.text !== "string" && input.typed_before === true && (["click", "double_click"].includes(action.type) || (action.type === "key" && String(action.key).toLowerCase() === "enter"))) reasons.push({ code: "submit_after_typing" });if (action.type === "type") {if (target.secure === true) reasons.push({ code: "secure_field" });if (looksLikeCredential(action.text)) reasons.push({ code: "credential_text" });@@ -391,6 +396,7 @@ export const ASK_REASONS = Object.freeze({other_window: "It would act in another window.",document_changed: "The window now shows a different page or document",after_takeover: "First action after you took over.",+ submit_after_typing: "It would click or press Enter right after typing, which may send what was typed.",});/**diff --git a/apps/server/src/computer-use/shared/portable-host.d.mts b/apps/server/src/computer-use/shared/portable-host.d.mtsindex 01a993de..bfb78b6d 100644--- a/apps/server/src/computer-use/shared/portable-host.d.mts+++ b/apps/server/src/computer-use/shared/portable-host.d.mts@@ -79,6 +79,8 @@ export type PortableHost = {control(input: { session_id: string; action: "stop" | "takeover" | "continue" | "hide" | "show" }): Promise<void>;stopAll(by?: SessionEvent["by"], event?: SessionEvent["event"]): Promise<void>;syntheticInput(withinMs?: number): Promise<boolean>;+ /** A physical key or button press within `withinMs` (not injected, not a remote-debugging click). */+ physicalInput(withinMs?: number, options?: { foregroundPid?: number | null }): Promise<boolean>;watchInput(): Promise<void>;state(): HostSessionState[];close(): Promise<void>;diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjsindex 5ca3bff3..0fe3c51d 100644--- a/apps/server/src/computer-use/shared/portable-host.mjs+++ b/apps/server/src/computer-use/shared/portable-host.mjs@@ -272,11 +272,15 @@ export async function createPortableHost({ native: firstNative, nativeFactory =try { spool.put({ bytes, mime, width: model.width, height: model.height, full, thumb: encodeJpeg(fitLongSide(model, 320), { quality: 70 }), ocr, masked }); } catch { /* the archive copy is best effort */ }}const ratio = model.width / raw.width;- const caret = raw.caret ? { x: raw.caret.x * ratio, y: raw.caret.y * ratio, width: raw.caret.width * ratio, height: raw.caret.height * ratio } : null;+ const scaled = (r) => ({ x: r.x * ratio, y: r.y * ratio, width: r.width * ratio, height: r.height * ratio });+ const caret = raw.caret ? scaled(raw.caret) : null;+ // What the host can read on screen (macOS: AX / text recognition), for the per-act rules only: never sent to the model.+ const texts = Array.isArray(raw.texts) ? raw.texts.filter((t) => t && typeof t.text === "string" && [t.x, t.y, t.width, t.height].every(Number.isFinite)).slice(0, 2000).map((t) => ({ ...scaled(t), text: t.text.slice(0, 200) })) : null;+ const secureFields = (Array.isArray(raw.secure_fields) ? raw.secure_fields : raw.secure ?? []).filter((r) => r && [r.x, r.y, r.width, r.height].every(Number.isFinite)).map(scaled);const id = randomUUID();const value = {id, at: Date.now(), window: w, sha, mime, bytes,- model: { width: model.width, height: model.height }, grid: luminanceGrid(model), caret,+ model: { width: model.width, height: model.height }, grid: luminanceGrid(model), caret, texts, secureFields,ref: { id, width: model.width, height: model.height, sha256: sha, mime, ...(full ? { full: { width: full.width, height: full.height, sha256: full.sha256 } } : {}), ocr, masked },};s.lastFrame = { sha, width: model.width, height: model.height, mime, data_url: `data:${mime};base64,${bytes.toString("base64")}`, at: now() };@@ -307,6 +311,17 @@ export async function createPortableHost({ native: firstNative, nativeFactory =return mcpResult(observeResult({ session_id: s.id, observation: obs.ref, window: windowRef(w), screen: s.screen, scale: w.bounds.width / obs.model.width, dpi: w.dpi, monitor_id: w.monitor, cursor: await native.call("cursor").catch(() => null), os, expires_in_ms: l.observation_ttl_seconds * 1000, actions_left: Math.max(0, l.actions_per_session - s.actions), seconds_left: Math.max(0, Math.ceil((s.expiresAt - Date.now()) / 1000)) }), includeImage ? { data: obs.bytes, mime: obs.mime } : null);}+ /** The on-screen text at (or within 24 px of) the action's point, and whether it is in a secure field. Unknown text: undefined. */+ function targetAt(observed, action) {+ const p = action.type === "drag" ? action.path.at(-1) : action.x !== undefined ? { x: action.x, y: action.y } : null;+ if (!p) return {};+ const inside = (r, pad) => p.x >= r.x - pad && p.x <= r.x + r.width + pad && p.y >= r.y - pad && p.y <= r.y + r.height + pad;+ const secure = (observed.secureFields ?? []).some((r) => inside(r, 0));+ if (!observed.texts) return secure ? { secure } : {};+ const near = observed.texts.filter((t) => inside(t, 24)).sort((a, b) => (inside(a, 0) ? 0 : 1) - (inside(b, 0) ? 0 : 1)).slice(0, 4);+ return { text: near.map((t) => t.text).join(" "), ...(secure ? { secure } : {}) };+ }+async function actTool(s, args) {const l = limits();const started = Date.now();@@ -349,11 +364,12 @@ export async function createPortableHost({ native: firstNative, nativeFactory =if (compare.stale) return finish({ ...common, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });}// Per-action approval by permission mode (host channel, never the engine's).- const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: {}, window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover });+ const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: targetAt(observed, action), window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover, typed_before: s.typedIn === String(current.id) });let approval = { asked: false };if (decision.ask) {// The same action with the same intent, allowed a moment ago (a retry after a fresh observation), is not asked again.- const key = JSON.stringify([action, args.intent]);+ // Keyed to the image too: the same screen, the same action, the same reason.+ const key = JSON.stringify([action, args.intent, observed.sha]);const remembered = s.allowed.get(key);if (remembered && remembered > Date.now()) {approval = { asked: false, reasons: decision.reasons.map((r) => r.code), by: "person" };@@ -390,6 +406,9 @@ export async function createPortableHost({ native: firstNative, nativeFactory =return finish({ ...common, approval, point, path, cursor_before: cursorBefore, ok: false, code, message: error.message, may_have_acted: action.type !== "wait", next: "computer_session_status" });}s.lastOperation = Date.now();+ // "Submit after typing" (Normal, when the host reads no text): typing arms it, a click or Enter after it asks.+ if (action.type === "type") s.typedIn = String(current.id);+ else if (["click", "double_click"].includes(action.type) || (action.type === "key" && action.key === "Enter")) s.typedIn = null;const cursorAfter = await native.call("cursor").catch(() => null);const done = { ...common, approval, point, path, cursor_before: cursorBefore, cursor_after: cursorAfter, ok: true, may_have_acted: action.type !== "wait" };if (!args.observe_after) return finish({ ...done, next: "computer_observe" });@@ -626,6 +645,17 @@ export async function createPortableHost({ native: firstNative, nativeFactory =}/** Whether the latest press (key or button) within `withinMs` was synthetic (injected by a program). */+ /**+ * Whether a physical key or button press (not injected by a program, not a remote-debugging+ * click that makes no OS input) came within `withinMs`: the person's own answer.+ */+ async function physicalInput(withinMs = 5000, { foregroundPid = null } = {}) {+ const value = await native.call("input-source").catch(() => null);+ if (!value || !value.at || value.synthetic || Date.now() - value.at >= withinMs) return false;+ // In the answering app's window (when the worker knows which app was in front).+ return !(foregroundPid && Number.isInteger(value.foreground_pid) && value.foreground_pid !== foregroundPid);+ }+async function syntheticInput(withinMs = 5000) {const value = await native.call("input-source").catch(() => null);return Boolean(value && value.synthetic && Date.now() - value.at < withinMs);@@ -642,5 +672,5 @@ export async function createPortableHost({ native: firstNative, nativeFactory =}update();- return { get availability() { return availability; }, os, connect, listen, control, stopAll, syntheticInput, watchInput, state: publicState, close };+ return { get availability() { return availability; }, os, connect, listen, control, stopAll, syntheticInput, physicalInput, watchInput, state: publicState, close };}diff --git a/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts b/apps/server/src/opencode-plugins/omnirush-mcp-policy.tsindex 7bd63483..51a98e3e 100644--- a/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts+++ b/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts@@ -65,14 +65,14 @@ function withScreenBashRules(permission: Record<string, unknown>): Record<string}/** Adds the left-out `computer-use` entry to the project's MCP report (Settings > MCP). */-function reportReserved(root: string): void {+function reportReserved(root: string, names: string[]): void {const dir = process.env.OMNIRUSH_MCP_REPORT_DIR;if (!dir) return;try {const file = reportPath(dir, root);const report: unknown = JSON.parse(readFileSync(file, "utf8"));if (!isRecord(report) || !Array.isArray(report.decisions)) return;- report.decisions = [...report.decisions.filter((d) => !(isRecord(d) && d.name === SERVER_ID)), { name: SERVER_ID, status: "blocked", code: "reserved", reason: "This name belongs to the app's built-in computer use; this entry is left out." }];+ report.decisions = [...report.decisions.filter((d) => !(isRecord(d) && names.includes(String(d.name)))), ...names.map((name) => ({ name, status: "blocked", code: "reserved", reason: "This name belongs to the app's built-in computer use; this entry is left out." }))];const temp = `${file}.${process.pid}.cu.tmp`;writeFileSync(temp, JSON.stringify(report), { mode: 0o600 });renameSync(temp, file);@@ -93,7 +93,24 @@ function withoutSubagentComputerTools(target: Record<string, unknown>): void {target.agent = agents;}-export default async function omnirushMcpPolicy(input?: { directory?: string }) {+/** Config names that are, or would make tool names of, the built-in server (`computer-use`, `computer-use_x` → `computer-use_x_act`). */+function reservedNames(mcp: Record<string, unknown>): string[] {+ return Object.keys(mcp).filter((name) => name.toLowerCase().startsWith(SERVER_ID));+}++type SessionClient = { session?: { get?: (options: { path: { id: string } }) => Promise<{ data?: { parentID?: unknown } } | undefined> } };++export default async function omnirushMcpPolicy(input?: { directory?: string; client?: SessionClient }) {+ // Sub-agents (sessions with a parent) never use computer use (one host lease per app).+ const parents = new Map<string, boolean>();+ const isSubagent = async (sessionID: unknown): Promise<boolean> => {+ if (typeof sessionID !== "string" || !sessionID || !input?.client?.session?.get) return false;+ if (!parents.has(sessionID)) {+ const found = await input.client.session.get({ path: { id: sessionID } }).catch(() => undefined);+ parents.set(sessionID, Boolean(found?.data?.parentID));+ }+ return parents.get(sessionID) === true;+ };const root = path.resolve(input?.directory || process.cwd());const hooks = createMcpPolicyHooks({directory: root,@@ -113,14 +130,14 @@ export default async function omnirushMcpPolicy(input?: { directory?: string })}}// The id `computer-use` belongs to the app's built-in server: a config entry with it is left out (and reported).- const reserved = target && isRecord(target.mcp) && Object.hasOwn(target.mcp, SERVER_ID);- if (target && reserved && isRecord(target.mcp)) {- const { [SERVER_ID]: _dropped, ...rest } = target.mcp;- target.mcp = rest;+ const reserved = target && isRecord(target.mcp) ? reservedNames(target.mcp) : [];+ if (target && reserved.length && isRecord(target.mcp)) {+ const mcp = target.mcp;+ target.mcp = Object.fromEntries(Object.entries(mcp).filter(([name]) => !reserved.includes(name)));}// Fails closed: an error while applying the list leaves no servers at all.await hooks.config(configInput, output);- if (reserved) reportReserved(root);+ if (reserved.length) reportReserved(root, reserved);const builtin = target ? computerUseBuiltins(process.env[COMPUTER_USE_DIR_ENV])[SERVER_ID] : null;if (target && builtin) {target.mcp = { ...(isRecord(target.mcp) ? target.mcp : {}), [SERVER_ID]: builtin };@@ -139,6 +156,11 @@ export default async function omnirushMcpPolicy(input?: { directory?: string })"experimental.chat.messages.transform": async (_input: unknown, output: { messages?: unknown[] }) => {if (Array.isArray(output?.messages)) pruneImages(output.messages, { match: (part: Record<string, unknown>) => isComputerTool(part.tool) });},- "tool.execute.before": hooks["tool.execute.before"],+ "tool.execute.before": async (toolInput: { tool?: unknown; args?: unknown; sessionID?: unknown }, toolOutput?: { args?: unknown }) => {+ if (typeof toolInput?.tool === "string" && toolInput.tool.startsWith(`${SERVER_ID}_`) && (await isSubagent(toolInput.sessionID))) {+ throw new Error("Computer use is not available to sub-agents. Ask the main agent to use it.");+ }+ await hooks["tool.execute.before"](toolInput, toolOutput);+ },};}