A desktop coding agent with free access to frontier models.
TypeScript
fix: computer use keeps the agent display apart from the person's desktop
ogsamrat552a868c77c2
diff --git a/apps/desktop/electron/browser-panel.mjs b/apps/desktop/electron/browser-panel.mjsindex 41e7afbf..85f47123 100644--- a/apps/desktop/electron/browser-panel.mjs+++ b/apps/desktop/electron/browser-panel.mjs@@ -877,8 +877,10 @@ export function createBrowserPanel({ getWindow, remoteDebugPort, onDeepLink, chereturn browserProxyState();}- app.on("login", (event, _webContents, _details, authInfo, callback) => {+ app.on("login", (event, webContents, _details, authInfo, callback) => {if (!authInfo?.isProxy || !browserProxy?.username) return;+ // The browser panel's proxy credentials are never given to computer use's own browser (its own sessions).+ if (webContents?.session !== browserSession) return;event.preventDefault();callback(browserProxy.username, browserProxy.password);});diff --git a/apps/desktop/electron/computer-use-agent-browser.mjs b/apps/desktop/electron/computer-use-agent-browser.mjsindex 2a8d9a2a..8d348278 100644--- a/apps/desktop/electron/computer-use-agent-browser.mjs+++ b/apps/desktop/electron/computer-use-agent-browser.mjs@@ -32,6 +32,7 @@ const ADDRESS = Object.freeze({ x: 110, right: SCREEN.width - 10 });const BLANK = "about:blank";const SEARCH = "https://duckduckgo.com/?q=";const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));+const CLIP_BINDING = "__omnirushClip";/** `promise`, or a rejection after `ms` (setup steps never hang the session). */const withTimeout = (promise, ms) => Promise.race([promise, new Promise((_, reject) => setTimeout(() => reject(new Error("timed out")), ms))]);const fail = (code, message) => Object.assign(new Error(message), { code });@@ -146,6 +147,39 @@ export function clipboardOf(name, modifiers = []) {return { c: "copy", x: "cut", v: "paste" }[String(name).toLowerCase()] ?? null;}+/**+ * Runs in every document of the agent's browser (main world, every frame): the page's own copy and cut+ * (document.execCommand, navigator.clipboard.write*) go to this browser's private clipboard through the+ * __omnirushClip binding, never to the person's clipboard; reading the clipboard, printing and native dialogs+ * are refused.+ */+export const PAGE_GUARD_JS = `(() => {+ const send = (t) => { try { window.__omnirushClip(String(t ?? "").slice(0, 1000000)); } catch {} };+ const fieldText = (el) => { try { return el && (el.tagName === "INPUT" || el.tagName === "TEXTAREA") && typeof el.selectionStart === "number" && el.selectionEnd > el.selectionStart ? el.value.slice(el.selectionStart, el.selectionEnd) : ""; } catch { return ""; } };+ // The focused field's selection, else a text field with a selection (copy helpers select a hidden textarea), else the page's.+ const selected = () => fieldText(document.activeElement) || [...document.querySelectorAll("textarea, input")].map(fieldText).find(Boolean) || String(getSelection() ?? "");+ const exec = Document.prototype.execCommand;+ Object.defineProperty(Document.prototype, "execCommand", { configurable: false, writable: false, value: function (command, ...rest) {+ const c = String(command).toLowerCase();+ if (c === "copy" || c === "cut") {+ let data = null;+ try { const dt = new DataTransfer(); const ev = new ClipboardEvent(c, { clipboardData: dt, bubbles: true, cancelable: true }); if (!(document.activeElement || document.body).dispatchEvent(ev)) data = dt.getData("text/plain"); } catch {}+ send(data ?? selected());+ if (c === "cut") { try { exec.call(this, "delete"); } catch {} }+ return true;+ }+ if (c === "paste") return false;+ return exec.call(this, command, ...rest);+ } });+ const cb = navigator.clipboard;+ if (cb) {+ const refuse = async () => { throw new DOMException("The clipboard is not available here.", "NotAllowedError"); };+ const write = async (items) => { for (const item of items ?? []) { if (item?.types?.includes("text/plain")) { send(await (await item.getType("text/plain")).text()); return; } } };+ for (const [name, fn] of [["writeText", async (t) => send(t)], ["write", write], ["readText", refuse], ["read", refuse]]) Object.defineProperty(cb, name, { configurable: false, writable: false, value: fn });+ }+ window.print = () => {}; window.alert = () => {}; window.confirm = () => true; window.prompt = () => null;+})();`;+/** The selected text in the page: inside a focused text field, else the document selection. */export const SELECTION_JS = "(() => { const a = document.activeElement; if (a && (a.tagName === 'INPUT' || a.tagName === 'TEXTAREA') && typeof a.selectionStart === 'number') return a.value.slice(a.selectionStart, a.selectionEnd); return String(getSelection() ?? ''); })()";@@ -196,7 +230,7 @@ function blit(out, bgra, width, rows, top) {// ------------------------------------------------------------ the backend/**- * Starts the agent's browser. `electron`: { BrowserWindow, session }; `glidePath(x0, y0, x1, y1)`: the shared+ * Starts the agent's browser. `electron`: { BrowserWindow, session, clipboard? }; `glidePath(x0, y0, x1, y1)`: the shared* worker's glide (virtual-worker.mjs). Returns the virtual host's backend.*/export async function startAgentBrowser({ electron, glidePath, url = null, versions = process.versions, wait = sleep }) {@@ -217,7 +251,7 @@ export async function startAgentBrowser({ electron, glidePath, url = null, versilet clip = ""; // this browser's own clipboardconst offscreen = (height) => new BrowserWindow({- show: false, width: SCREEN.width, height, useContentSize: true, frame: false, skipTaskbar: true, focusable: false, paintWhenInitiallyHidden: true,+ show: false, width: SCREEN.width, height, useContentSize: true, frame: false, skipTaskbar: true, focusable: false, paintWhenInitiallyHidden: true, fullscreenable: false,webPreferences: { offscreen: { useSharedTexture: false, deviceScaleFactor: 1 }, partition, sandbox: true, contextIsolation: true, nodeIntegration: false, backgroundThrottling: false, spellcheck: false, devTools: false },});const latest = (wc, view) => { wc.setFrameRate?.(30); wc.on("paint", (_event, _dirty, image) => { view.image = image; }); };@@ -245,7 +279,14 @@ export async function startAgentBrowser({ electron, glidePath, url = null, versireturn drawing;};- const cdp = (tab, method, params = {}) => tab.window.webContents.debugger.sendCommand(method, params);+ const cdp = (tab, method, params = {}, sessionId = undefined) => (sessionId ? tab.window.webContents.debugger.sendCommand(method, params, sessionId) : tab.window.webContents.debugger.sendCommand(method, params));+ /** An out-of-process frame: the clipboard binding, the page guard, its own frames, then it may run. */+ async function guardTarget(tab, sessionId) {+ for (const [method, params] of [["Runtime.enable", {}], ["Runtime.addBinding", { name: CLIP_BINDING }], ["Page.enable", {}], ["Page.addScriptToEvaluateOnNewDocument", { source: PAGE_GUARD_JS }], ["Target.setAutoAttach", { autoAttach: true, waitForDebuggerOnStart: true, flatten: true }]]) {+ await withTimeout(cdp(tab, method, params, sessionId), 3000).catch(() => undefined);+ }+ await cdp(tab, "Runtime.runIfWaitingForDebugger", {}, sessionId).catch(() => undefined);+ }async function openTab(target, { activate = true } = {}) {const window = offscreen(PAGE_H);@@ -262,16 +303,30 @@ export async function startAgentBrowser({ electron, glidePath, url = null, versiwc.on("did-start-loading", () => { tab.loading = true; void draw(); });wc.on("did-stop-loading", () => { tab.loading = false; void draw(); });wc.on("render-process-gone", () => { tab.title = "This page crashed"; void draw(); });+ // No client certificate and no credentials from the person's other sessions: both are cancelled.+ wc.on("select-client-certificate", (event, _url, _list, callback) => { event.preventDefault(); callback(); });+ wc.on("login", (event, _details, _auth, callback) => { event.preventDefault(); callback(); });// A committed document first: DevTools commands on a window that never loaded anything do not answer.await withTimeout(window.loadURL(BLANK), 5000).catch(() => undefined);wc.debugger.attach("1.3");- wc.debugger.on("message", (_event, method, params) => {+ wc.debugger.on("message", (_event, method, params, sessionId) => {// No native dialog or file picker ever reaches the person's screen.- if (method === "Page.javascriptDialogOpening") void cdp(tab, "Page.handleJavaScriptDialog", { accept: true }).catch(() => undefined);+ if (method === "Page.javascriptDialogOpening") void cdp(tab, "Page.handleJavaScriptDialog", { accept: true }, sessionId).catch(() => undefined);+ // The page's own copy (execCommand, navigator.clipboard) lands in this browser's clipboard.+ if (method === "Runtime.bindingCalled" && params?.name === CLIP_BINDING && typeof params.payload === "string" && params.payload) clip = params.payload.slice(0, 1_000_000);+ // Frames in other processes get the same guard before they run.+ if (method === "Target.attachedToTarget" && params?.sessionId) void guardTarget(tab, params.sessionId);+ // An HTML drag started by a page (draggable items): carried through as DevTools drag events.+ if (method === "Input.dragIntercepted") tab.dragData = params?.data ?? null;});await withTimeout(cdp(tab, "Page.enable"), 3000).catch(() => undefined);await withTimeout(cdp(tab, "Page.setInterceptFileChooserDialog", { enabled: true }), 3000).catch(() => undefined);- await withTimeout(cdp(tab, "Page.addScriptToEvaluateOnNewDocument", { source: "window.alert=()=>{};window.confirm=()=>true;window.prompt=()=>null;" }), 3000).catch(() => undefined);+ // Runtime events (the clipboard binding's calls) arrive only with the domain on.+ await withTimeout(cdp(tab, "Runtime.enable"), 3000).catch(() => undefined);+ await withTimeout(cdp(tab, "Runtime.addBinding", { name: CLIP_BINDING }), 3000).catch(() => undefined);+ await withTimeout(cdp(tab, "Page.addScriptToEvaluateOnNewDocument", { source: PAGE_GUARD_JS }), 3000).catch(() => undefined);+ await withTimeout(cdp(tab, "Target.setAutoAttach", { autoAttach: true, waitForDebuggerOnStart: true, flatten: true }), 3000).catch(() => undefined);+ await withTimeout(cdp(tab, "Input.setInterceptDrags", { enabled: true }), 3000).catch(() => undefined);await withTimeout(cdp(tab, "Emulation.setFocusEmulationEnabled", { enabled: true }), 3000).catch(() => undefined);tabs.push(tab);if (activate) { active = tabs.length - 1; address.focused = target === null; address.text = ""; address.selected = false; }@@ -413,11 +468,25 @@ export async function startAgentBrowser({ electron, glidePath, url = null, versiif (i > 0) await key("Enter");if (!lines[i]) continue;if (address.focused) await typeAddress(lines[i]);- else if (current()) await cdp(current(), "Input.insertText", { text: lines[i] });+ else if (current()) {+ // Like a person: one key event per character, the character as its text (any script or emoji, no IME involved).+ for (const ch of lines[i]) {+ await cdp(current(), "Input.dispatchKeyEvent", { type: "keyDown", key: ch, text: ch, unmodifiedText: ch });+ await cdp(current(), "Input.dispatchKeyEvent", { type: "keyUp", key: ch });+ }+ }}}+ /** Linux: a selection in the agent's pages must not become the person's middle-click selection (PRIMARY). */async function act(action) {+ const primary = process.platform === "linux" && electron.clipboard ? electron.clipboard.readText("selection") : null;+ try { return await actNow(action); } finally {+ if (primary !== null && electron.clipboard.readText("selection") !== primary) electron.clipboard.writeText(primary, "selection");+ }+ }++ async function actNow(action) {if (closed) throw fail("session_unavailable", "The agent's browser has closed.");const t = action.type;if (t === "wait") return wait(action.ms ?? 0);@@ -441,9 +510,16 @@ export async function startAgentBrowser({ electron, glidePath, url = null, versiconst start = hitTest(first.x, first.y, tabs.length);if (start.area !== "page") return undefined;held = "left";+ const tab = current();+ if (tab) tab.dragData = null;await mouse("mousePressed", start, { button: "left", buttons: 1, clickCount: 1 });for (const point of rest) await glide(point.x, Math.max(CHROME_H, point.y), { button: "left", buttons: 1 });const end = hitTest(pointer.x, Math.max(CHROME_H, pointer.y), tabs.length);+ if (tab?.dragData) {+ // A page drag (not a text selection): enter, over and drop where the agent's cursor stopped.+ for (const type of ["dragEnter", "dragOver", "drop"]) await cdp(tab, "Input.dispatchDragEvent", { type, x: end.x, y: end.y, data: tab.dragData, modifiers: 0 }).catch(() => undefined);+ tab.dragData = null;+ }await mouse("mouseReleased", end, { button: "left", buttons: 0, clickCount: 1 });held = null;return undefined;diff --git a/apps/desktop/electron/computer-use-agent-browser.test.mjs b/apps/desktop/electron/computer-use-agent-browser.test.mjsindex 87e0bab1..1c4a4ed9 100644--- a/apps/desktop/electron/computer-use-agent-browser.test.mjs+++ b/apps/desktop/electron/computer-use-agent-browser.test.mjs@@ -171,11 +171,13 @@ test("agent browser: typing inserts text (IME-safe), newlines press Enter, keysconst b = await start({ url: "https://a.test/" });const page = b.active().webContents;page.commands.length = 0;- await b.act({ type: "type", text: "héllo 世界 👋" });- assert.deepEqual(page.commands, [["Input.insertText", { text: "héllo 世界 👋" }]]);+ await b.act({ type: "type", text: "hé世👋" });+ // One key event per character, the character as its text (surrogate pairs kept whole).+ assert.deepEqual(page.commands.filter(([, p]) => p.type === "keyDown").map(([, p]) => p.text), ["h", "é", "世", "👋"]);+ assert.equal(page.commands.filter(([, p]) => p.type === "keyUp").length, 4);page.commands.length = 0;await b.act({ type: "type", text: "a\nb" });- assert.deepEqual(page.commands.map(([method, p]) => (method === "Input.insertText" ? p.text : `${p.type}:${p.key}`)), ["a", "keyDown:Enter", "keyUp:Enter", "b"]);+ assert.deepEqual(page.commands.map(([, p]) => `${p.type}:${p.key}`), ["keyDown:a", "keyUp:a", "keyDown:Enter", "keyUp:Enter", "keyDown:b", "keyUp:b"]);// Copy and paste use the browser's own clipboard: the selection is read from the page, never the OS clipboard.const send = page.debugger.sendCommand;page.debugger.sendCommand = async (method, params) => (method === "Runtime.evaluate" ? (page.commands.push([method, params]), { result: { value: "7Q-3381" } }) : send(method, params));@@ -185,7 +187,7 @@ test("agent browser: typing inserts text (IME-safe), newlines press Enter, keysassert.ok(!page.commands.some(([, p]) => p?.commands), "no OS clipboard command");page.commands.length = 0;await b.act({ type: "key", key: "v", modifiers: ["meta"] });- assert.deepEqual(page.commands, [["Input.insertText", { text: "7Q-3381" }]]);+ assert.deepEqual(page.commands, [["Input.insertText", { text: "7Q-3381" }]], "paste inserts the private clipboard");await b.backend.close();});@@ -240,3 +242,29 @@ test("agent browser: tabs from the chrome and the chords; window.open opens a taassert.match(b.fake.windows[0].webContents.scripts.at(-1), /New tab/);await b.backend.close();});++test("agent browser: every document gets the page guard (private clipboard binding, no print or dialogs), frames in other processes too", async () => {+ const { PAGE_GUARD_JS } = await import("./computer-use-agent-browser.mjs");+ const b = await start({ url: "https://a.test/" });+ const page = b.active().webContents;+ const setup = page.commands.map(([method, p]) => [method, p?.name ?? p?.source?.slice(0, 10) ?? null]);+ assert.ok(setup.some(([m, n]) => m === "Runtime.addBinding" && n === "__omnirushClip"));+ assert.ok(setup.some(([m]) => m === "Page.addScriptToEvaluateOnNewDocument") && page.commands.find(([m]) => m === "Page.addScriptToEvaluateOnNewDocument")[1].source === PAGE_GUARD_JS);+ assert.ok(page.commands.some(([m, p]) => m === "Target.setAutoAttach" && p.autoAttach && p.flatten && p.waitForDebuggerOnStart));+ assert.ok(page.commands.some(([m, p]) => m === "Input.setInterceptDrags" && p.enabled));+ // The page's own Copy button (execCommand / navigator.clipboard) reaches the binding: then paste inserts it.+ page.debugger.emit("message", {}, "Runtime.bindingCalled", { name: "__omnirushClip", payload: "from the page" });+ page.commands.length = 0;+ await b.act({ type: "key", key: "v", modifiers: ["ctrl"] });+ assert.deepEqual(page.commands, [["Input.insertText", { text: "from the page" }]]);+ // A frame in another process: guarded and only then resumed.+ page.commands.length = 0;+ page.debugger.emit("message", {}, "Target.attachedToTarget", { sessionId: "S1", targetInfo: { type: "iframe" } });+ await new Promise((r) => setTimeout(r, 20));+ const methods = page.commands.map(([m]) => m);+ assert.deepEqual(methods.slice(-1), ["Runtime.runIfWaitingForDebugger"]);+ assert.ok(methods.indexOf("Page.addScriptToEvaluateOnNewDocument") < methods.indexOf("Runtime.runIfWaitingForDebugger"));+ // The guard itself: copy/cut/clipboard writes go to the binding, reads are refused, print is a no-op.+ for (const needle of ["execCommand", "writeText", "readText", "window.print = () => {}", "__omnirushClip"]) assert.ok(PAGE_GUARD_JS.includes(needle), needle);+ await b.backend.close();+});diff --git a/apps/desktop/electron/computer-use.mjs b/apps/desktop/electron/computer-use.mjsindex ea296853..c6458f82 100644--- a/apps/desktop/electron/computer-use.mjs+++ b/apps/desktop/electron/computer-use.mjs@@ -21,7 +21,7 @@ import { createRequire } from "node:module";import path from "node:path";import { fileURLToPath, pathToFileURL } from "node:url";-import { app, BrowserWindow, screen, session } from "electron";+import { app, BrowserWindow, clipboard, screen, session } from "electron";import { startAgentBrowser } from "./computer-use-agent-browser.mjs";import { testAnswersAllowed } from "./computer-use-answers.mjs";import { createComputerUseHost } from "./computer-use-host.mjs";@@ -193,7 +193,7 @@ async function startHost() {// Linux with Xvfb (and a browser for it): a whole display of its own; elsewhere the agent-only browser in this app.startBackend: (name, options) => (process.platform === "linux" && virtualModule.which("Xvfb") && (!options.browser || virtualModule.findBrowser())? virtualModule.startXvfbBackend({ name, ...options, nodePath: process.execPath, nodeEnv: { ELECTRON_RUN_AS_NODE: "1" }, koffiPath: koffiPath() })- : startAgentBrowser({ electron: { BrowserWindow, session }, glidePath: virtualWorker.glidePath, url: options.url })),+ : startAgentBrowser({ electron: { BrowserWindow, session, clipboard }, glidePath: virtualWorker.glidePath, url: options.url })),settings: hostSettings, context: hostContext, spool, onEvent, onState, onFrame, onCursor,});const callbacks = { spool, settings: hostSettings, context: hostContext, refresh: hostRefresh, virtual, onEvent, onState, onFrame };diff --git a/apps/server/src/computer-use/contract-policy.test.ts b/apps/server/src/computer-use/contract-policy.test.tsindex 3d961663..9ba5cfcc 100644--- a/apps/server/src/computer-use/contract-policy.test.ts+++ b/apps/server/src/computer-use/contract-policy.test.ts@@ -328,11 +328,9 @@ describe("policy: approvals per permission mode", () => {expect(normalizeLimits({}).approval_timeout_seconds).toBe(120);});- test("Allow for this chat: until the chat ends by default; the server's value is an upper bound", () => {- expect(DEFAULT_LIMITS.chat_approval_seconds).toBe(0);- expect(normalizeLimits({ chat_approval_seconds: 1800 }).chat_approval_seconds).toBe(1800);- expect(normalizeLimits({ chat_approval_seconds: 0 }).chat_approval_seconds).toBe(0);- expect(normalizeLimits({}).chat_approval_seconds).toBe(0);+ test("no grant limits are kept any more (computer use never asks)", () => {+ expect("chat_approval_seconds" in DEFAULT_LIMITS).toBe(false);+ expect("chat_approval_seconds" in normalizeLimits({ chat_approval_seconds: 1800 })).toBe(false);});test("shell screen capture asks; sub-agents get no computer tools", () => {diff --git a/apps/server/src/computer-use/desktop.test.ts b/apps/server/src/computer-use/desktop.test.tsindex b339649a..46d5cae0 100644--- a/apps/server/src/computer-use/desktop.test.ts+++ b/apps/server/src/computer-use/desktop.test.ts@@ -1,6 +1,7 @@// Desktop computer use on the server side: the switch, the built-in MCP entry// (reserved id), the engine plugin's hooks, the media lane and its store.import { afterAll, describe, expect, test } from "bun:test";+import { REAL_SCREEN_DENY_RULES } from "./shared/policy.mjs";import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";import os from "node:os";import path from "node:path";@@ -85,7 +86,7 @@ describe("switch and built-in entry", () => {});});-describe("routes: the chat a grant is kept for, and the policy read before an open", () => {+describe("routes: the chat in use, and the policy read before an open", () => {test("state names a chat only when exactly one prompted chat runs; refresh reads the server now (max_age_ms) and says whether it did", async () => {const routes: Route[] = [];const state = tmp();@@ -93,7 +94,7 @@ describe("routes: the chat a grant is kept for, and the policy read before an oplet running: string[] | null = null;const ages: number[] = [];let failRead = false;- registerComputerUseRoutes({ routes, stateDir: () => state, jsonResponse: (d, status = 200) => new Response(JSON.stringify(d), { status }), readJsonBody: async (r) => r.json(), approvalMode: async () => "guarded", readCapabilities: async (maxAgeMs) => { ages.push(maxAgeMs); if (failRead) throw new Error("offline"); return { computer_use: { enabled: true, limits: { chat_approval_seconds: 600 } } }; }, applyEngine: async () => undefined, chats, runningSessions: async () => running, recordTrace: () => undefined });+ registerComputerUseRoutes({ routes, stateDir: () => state, jsonResponse: (d, status = 200) => new Response(JSON.stringify(d), { status }), readJsonBody: async (r) => r.json(), approvalMode: async () => "guarded", readCapabilities: async (maxAgeMs) => { ages.push(maxAgeMs); if (failRead) throw new Error("offline"); return { computer_use: { enabled: true, limits: { session_seconds: 600 } } }; }, applyEngine: async () => undefined, chats, runningSessions: async () => running, recordTrace: () => undefined });const hit = async (method: string, p: string, body?: unknown) => {const route = routes.find((r) => r.method === method && r.regex.test(p));if (!route) throw new Error("no route");@@ -105,17 +106,18 @@ describe("routes: the chat a grant is kept for, and the policy read before an opexpect((await hit("GET", "/computer-use/state")).chat).toBeNull();running = ["ses_a"];expect((await hit("GET", "/computer-use/state")).chat).toBe("ses_a");- // Switching chat: the new one is named, so a grant from ses_a is not reused there.+ // Switching chat: the new one is named.chats.mark("ses_b"); running = ["ses_b"];expect((await hit("GET", "/computer-use/state")).chat).toBe("ses_b");- // Both running: no chat (asked every time).+ // Both running: no chat.running = ["ses_a", "ses_b"];expect((await hit("GET", "/computer-use/state")).chat).toBeNull();// Before an open the host asks with max_age_ms 0; the new bound is in force at once.const read = await hit("POST", "/computer-use/refresh", { max_age_ms: 0 });expect(read.read).toBe(true);expect(ages.at(-1)).toBe(2_000);- expect((read.setting as { limits: { chat_approval_seconds: number } }).limits.chat_approval_seconds).toBe(600);+ expect((read.setting as { limits: { session_seconds: number } }).limits.session_seconds).toBe(600);+ expect(Object.keys(read.setting as Record<string, unknown>)).not.toContain("grant_epoch");expect((await hit("POST", "/computer-use/refresh", {})).read).toBe(true);expect(ages.at(-1)).toBe(60_000);failRead = true;@@ -152,7 +154,8 @@ describe("engine MCP plugin", () => {const mcp = config.mcp as Record<string, { command: string[] }>;expect(Object.keys(mcp)).toEqual(["computer-use"]);expect(mcp["computer-use"]?.command).toEqual([process.execPath, "/x/relay.mjs", "/x/c.json"]);- expect(Object.entries(config.permission as Record<string, string>).at(-1)).toEqual(["computer-use_*", "allow"]);+ expect((config.permission as Record<string, string>)["computer-use_*"]).toBe("allow");+ expect((config.permission as { bash: Record<string, string> }).bash["*computer real-screen*"]).toBe("deny");// The built-in server gets the allowlist's environment too: the engine's tokens empty for it, its own kept.const builtinEnv = (config.mcp as Record<string, { environment: Record<string, string> }>)["computer-use"]!.environment;expect(builtinEnv.OMNIRUSH_COMPUTER_USE_COMMAND).toBe("");@@ -191,16 +194,17 @@ describe("engine MCP plugin", () => {test("off: nothing is added and the config is as before", async () => {const state = tmp();const { config } = await run({ OMNIRUSH_COMPUTER_USE_DIR: computerUseDir(state) }, { permission: { "*": "ask", bash: { "*": "allow", "rm -r*": "ask" } } });- expect(config).toEqual({ mcp: {}, permission: { "*": "ask", bash: { "*": "allow", "rm -r*": "ask" } } });+ // As before, plus only the real-screen denies (every mode, computer use on or off).+ expect(config).toEqual({ mcp: {}, permission: { "*": "ask", bash: { "*": "allow", "rm -r*": "ask", ...REAL_SCREEN_DENY_RULES.bash }, edit: { ...REAL_SCREEN_DENY_RULES.edit } } });});- test("on in Auto: the shell rules stay a single allow", async () => {+ test("on in Auto: the shell rules stay a single allow (plus the real-screen denies)", async () => {const state = tmp();const dir = computerUseDir(state);writeHostFile(dir, { command: [process.execPath, "/x/relay.mjs", "/x/c.json"], environment: {} });const pinnedEnv = { OMNIRUSH_COMPUTER_USE_COMMAND: JSON.stringify({ command: [process.execPath, "/x/relay.mjs", "/x/c.json"], environment: {} }) };const { config } = await run({ ...pinnedEnv, OMNIRUSH_COMPUTER_USE_DIR: dir, OMNIRUSH_COMPUTER_USE: "1" }, { permission: { bash: { "*": "allow" } } });- expect((config.permission as { bash: Record<string, string> }).bash).toEqual({ "*": "allow" });+ expect((config.permission as { bash: Record<string, string> }).bash).toEqual({ "*": "allow", ...REAL_SCREEN_DENY_RULES.bash });});test("model context pruning touches only computer tool parts", async () => {diff --git a/apps/server/src/computer-use/routes.ts b/apps/server/src/computer-use/routes.tsindex 234a49a8..92bfd599 100644--- a/apps/server/src/computer-use/routes.ts+++ b/apps/server/src/computer-use/routes.ts@@ -54,7 +54,7 @@ export function registerComputerUseRoutes(deps: ComputerUseRouteDeps): void {const chat = deps.chats.only(await deps.runningSessions().catch(() => null));return {enabled: setting.enabled,- setting: { enabled: setting.enabled, real_screen: setting.real_screen, virtual_display: setting.virtual_display, limits: setting.limits, deny_apps: setting.deny_apps, models: setting.models, message: setting.message, ask_send_delete: setting.ask_send_delete === true, grant_epoch: setting.grant_epoch ?? null, chat_bound_read: setting.chat_bound_read === true, reason: setting.reason, source: setting.source },+ setting: { enabled: setting.enabled, real_screen: setting.real_screen, virtual_display: setting.virtual_display, limits: setting.limits, deny_apps: setting.deny_apps, models: setting.models, message: setting.message, ask_send_delete: setting.ask_send_delete === true, reason: setting.reason, source: setting.source },mode,dir,spool: mediaSpoolDir(deps.stateDir()),diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256index fb3c3c76..f9fab3b3 100644--- a/apps/server/src/computer-use/shared/PARITY.sha256+++ b/apps/server/src/computer-use/shared/PARITY.sha256@@ -1,16 +1,16 @@# Computer use (contract v2, policy, media refs, image codecs, Windows/X11 host, the agent's own display): the same files in omnirush-cli (src/computer-use/) and omnirush-gui (apps/server/src/computer-use/shared/).# A change to one must be ported to the other and this list updated in both.-6ab3fdfbf466bdf24b748b678579a4113f46574ef1e0edb79e4ca9d8d9b56dfe contract.mjs-28d21d8624b75221c6420ff8b8366f743d69daebc423b594b8673d34b9b50aa6 contract.d.mts-f7fbd70ed4e98beeea6100b1b46125c6d7d8e76c9cc6ff5fa6fb49740de51269 policy.mjs-6d3f82f1b69a7b95d1329b7df033347e42e002c3e8637eef522bcf4b4a215be2 policy.d.mts+da1cc6c26f3bf6758d3e2ce70e87bce383f68ee0ec6c85bbdc33b1f9a6c407fa contract.mjs+30cfcead85a5b2b54005ba6a0a67a43d9950de944b4615861ce9342bf043d4a4 contract.d.mts+0ad2f6d4827deb710251b6aeb5dd325dff9637cb4129fb8168fe1e9345186938 policy.mjs+f3bcc7ce814a5f2a6566f8b558b53a429611e26cc4c237f43c55bd70312eb5c2 policy.d.mtsda515baa8aec2b533685d20c2be7ae007acc6b116db0386f8180e6779cb80f5c image.mjs7c1a3224b7efd215656a94c38cf3152fe7a2444f25a6162988528978acf5458d image.d.mtse47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjseabdf78c56733577d05aad490d9d7d5ece6c22e9b3ea9e6e9803d1a4a546fdd4 media.d.mtsa0bd964a5d0c7c5e26b4ce4ed8e8300f80a0a3a9da5d7f49c36f6ac2c907e934 native-worker.mjs-028a47b069e8d3936cda21b1ff43fb2bc95ad7223778a135f4d3187031b5435e portable-host.mjs+d7414bbbb57a2d18ab72fce26e900444106051ce03f28d468b6139388f453bbb portable-host.mjs40cc047cb02f4c50ac98508bab9c0f4e1ee80c1d40b4b05a472a11ce01e8a153 portable-host.d.mts-57ad5fdb1ea7b42e99b99733f0f4ee86288e34aaa60f538813bf80f93e9062f6 virtual-host.mjs+5e61fb875cd1b984c3daaaae6587d47485c90a97c4c769fecc38038166b74fcb virtual-host.mjs5bee4b675171a7cc8f02f172ba4b4d57171bb8ee27631974a5bfe33ed0bf58c8 virtual-host.d.mts-d56657ce6128dfc4aa12d630e70a854b765bf17fe07e0b66cdc6a1beb2b99281 virtual-worker.mjs+fc8bb13fd77f224d5a8a03d618e7253f877d4d678c21d245cd5fe91d45df097a virtual-worker.mjsdiff --git a/apps/server/src/computer-use/shared/contract.d.mts b/apps/server/src/computer-use/shared/contract.d.mtsindex ebf8c9ff..09bb1e16 100644--- a/apps/server/src/computer-use/shared/contract.d.mts+++ b/apps/server/src/computer-use/shared/contract.d.mts@@ -29,7 +29,6 @@ export type Limits = {full_long_side: number;observation_ttl_seconds: number;approval_timeout_seconds: number;- chat_approval_seconds: number;media_full_bytes: number;media_model_bytes: number;};diff --git a/apps/server/src/computer-use/shared/contract.mjs b/apps/server/src/computer-use/shared/contract.mjsindex 8ba8c797..33d7bd89 100644--- a/apps/server/src/computer-use/shared/contract.mjs+++ b/apps/server/src/computer-use/shared/contract.mjs@@ -61,9 +61,8 @@ export const DEFAULT_LIMITS = Object.freeze({model_long_side: 1280,full_long_side: 3840,observation_ttl_seconds: 15,+ // How long Continue (after the person's own input paused the real screen) waits for the person.approval_timeout_seconds: 120,- // "Allow for this chat": 0 = until the chat ends; a server value above 0 is an upper bound.- chat_approval_seconds: 0,media_full_bytes: 300 * 1024 * 1024,media_model_bytes: 200 * 1024 * 1024,});@@ -80,7 +79,6 @@ export const LIMIT_RANGES = Object.freeze({// At most 120 s: a question must end inside the engine's computer-use call timeout (CLI 150 s, less its// 7 s policy read; desktop 300 s), or a late answer would come with nobody waiting for it.approval_timeout_seconds: [10, 120],- chat_approval_seconds: [0, 30 * 86400],media_full_bytes: [0, 2 * 1024 * 1024 * 1024],media_model_bytes: [0, 2 * 1024 * 1024 * 1024],});@@ -101,7 +99,7 @@ export const INSTRUCTIONS = ["The person's own screen (other app_ids from computer_discover) is available only when the person turned it on in OmniRush's settings; it also runs without questions, so act there only for what the person asked.","Work in steps: computer_observe, then computer_act with the observation_id you just got (each one works once and expires after a few seconds). computer_act waits for the screen to settle and returns the next observation when observe_after is true (the default), so act again with that one.","Coordinates are pixels of the latest image you were given, measured from its top-left corner. Your cursor is drawn in the image (a purple arrow tagged AGENT).",- "Use the keyboard like a person: ctrl+l focuses the address bar, ctrl+t opens a tab, ctrl+tab / ctrl+shift+tab switch tabs, ctrl+w closes one, alt+tab switches windows.",+ "Use the keyboard like a person: ctrl+l focuses the address bar, ctrl+t opens a tab, ctrl+tab / ctrl+shift+tab switch tabs, ctrl+w closes one, alt+tab switches windows. To run programs, use your shell tool, not the screen.","Give every computer_act an intent: one short sentence saying why, in your words. It is recorded and shown to the person watching.","On the person's screen, their own mouse or keyboard input pauses control until they choose Continue; check computer_session_status and do not try to resume yourself.","Never retry an action whose result says may_have_acted: true; observe and check what happened.",diff --git a/apps/server/src/computer-use/shared/policy.d.mts b/apps/server/src/computer-use/shared/policy.d.mtsindex 5d4076bd..5808b5fc 100644--- a/apps/server/src/computer-use/shared/policy.d.mts+++ b/apps/server/src/computer-use/shared/policy.d.mts@@ -18,12 +18,8 @@ export type ComputerUseBlock = {deny_apps: DenyEntry[];limits: Limits;message: string | null;- /** The send/delete asks in Normal (off unless the server says true). */+ /** The desktop built-in browser tool's send/delete asks (off unless the server says true). */ask_send_delete: boolean;- /** The server's grant epoch (null: not sent): a grant is reused only under the epoch it was given in. */- grant_epoch: number | null;- /** Whether the server sent limits.chat_approval_seconds itself (false: no grant is reused). */- chat_bound_read: boolean;};export type ComputerUseSetting = ComputerUseBlock & {source: "env" | "server" | "default";@@ -39,6 +35,8 @@ export const REAL_SCREEN_DEFAULT: boolean;export const REAL_SCREEN_SETTING: string;export const REAL_SCREEN_ENV: string;export const REAL_SCREEN_OFF_MESSAGE: string;+export const REAL_SCREEN_FILE: string;+export const REAL_SCREEN_DENY_RULES: Readonly<{ bash: Readonly<Record<string, "deny">>; edit: Readonly<Record<string, "deny">> }>;export function realScreenOptIn(options?: { env?: Record<string, string | undefined>; saved?: unknown }): boolean;export type SessionRules = { open: true; asks: false } | { open: false; asks: false; code: "virtual_unavailable" | "real_screen_unavailable" | "real_screen_off"; message: string };export function sessionRules(options?: { display?: "virtual" | "real"; server?: { real_screen?: boolean; virtual_display?: boolean }; optIn?: boolean }): SessionRules;diff --git a/apps/server/src/computer-use/shared/policy.mjs b/apps/server/src/computer-use/shared/policy.mjsindex 0029ba31..32696edf 100644--- a/apps/server/src/computer-use/shared/policy.mjs+++ b/apps/server/src/computer-use/shared/policy.mjs@@ -41,11 +41,11 @@ function text(value, limit = 400) {* missing or malformed block is off. The server already applied* `rollout_percent` (and the account) to `enabled`.* Shape: { enabled, clients, os, models, real_screen, virtual_display,- * deny_apps, limits, message, ask_send_delete, grant_epoch, chat_bound_read }+ * deny_apps, limits, message, ask_send_delete }* (null lists: no restriction).*/export function normalizeComputerUse(block) {- if (!isRecord(block)) return { enabled: false, clients: null, os: null, models: null, real_screen: false, virtual_display: false, deny_apps: [], limits: normalizeLimits(null), message: null, ask_send_delete: false, grant_epoch: null, chat_bound_read: false };+ if (!isRecord(block)) return { enabled: false, clients: null, os: null, models: null, real_screen: false, virtual_display: false, deny_apps: [], limits: normalizeLimits(null), message: null, ask_send_delete: false };const denyApps = Array.isArray(block.deny_apps)? block.deny_apps.flatMap((item) => {if (typeof item === "string" && item.trim()) return [{ match: item.trim().slice(0, 200), reason: null }];@@ -63,13 +63,8 @@ export function normalizeComputerUse(block) {deny_apps: denyApps,limits: normalizeLimits(block.limits),message: text(block.message),- // The send/delete asks in Normal (actApproval options.sendDelete): only an explicit true turns them on.+ // The desktop's built-in browser tool only (actApproval options.sendDelete): only an explicit true turns them on.ask_send_delete: block.ask_send_delete === true,- // "Allow for this chat" is reused only under the epoch it was given in (the server bumps it when the- // switch or the bound changes) and only when the server sent the bound itself: else null / false.- grant_epoch: Number.isSafeInteger(block.grant_epoch) && block.grant_epoch >= 0 ? block.grant_epoch : null,- // (A block normalized here once already keeps its answer: its limits are filled in by then.)- chat_bound_read: block.chat_bound_read === false ? false : isRecord(block.limits) && typeof block.limits.chat_approval_seconds === "number" && Number.isFinite(block.limits.chat_approval_seconds),};}@@ -110,11 +105,22 @@ export function offMessage(setting) {/** The person's real screen is off until the person turns it on (Settings / `omnirush computer real-screen on`). */export const REAL_SCREEN_DEFAULT = false;-/** The settings key both apps save the person's choice under. */+/** The key both apps save the person's choice under, in <state or profile>/computer-use/real-screen.json. */export const REAL_SCREEN_SETTING = "computerUseRealScreen";+export const REAL_SCREEN_FILE = "real-screen.json";/** OMNIRUSH_COMPUTER_USE_REAL_SCREEN=1/0 overrides the saved choice (support, tests). */export const REAL_SCREEN_ENV = "OMNIRUSH_COMPUTER_USE_REAL_SCREEN";-export const REAL_SCREEN_OFF_MESSAGE = "Computer use on your own screen is off. The agent works on its own display instead (app_id \"virtual:main\"). The person can turn the real screen on in Settings (desktop) or with `omnirush computer real-screen on` (CLI).";+export const REAL_SCREEN_OFF_MESSAGE = "Computer use on the person's own screen is off. Work on your own display instead (app_id \"virtual:main\"). Only the person can change this setting; never change it yourself and never ask to.";++/**+ * Engine permission rules for every mode (computer use on or off): the agent can neither run the command that+ * turns the real screen on nor write the setting. Appended after the mode's own rules (the last match wins); the+ * engine's matcher treats \\ as / (Windows paths match too).+ */+export const REAL_SCREEN_DENY_RULES = Object.freeze({+ bash: Object.freeze({ "*computer real-screen*": "deny", "*computerUseRealScreen*": "deny", "*OMNIRUSH_COMPUTER_USE_REAL_SCREEN*": "deny", "*computer-use/real-screen.json*": "deny" }),+ edit: Object.freeze({ "*computer-use/real-screen.json": "deny" }),+});/** Whether the person turned the real screen on: the environment first, then the saved setting, else REAL_SCREEN_DEFAULT. */export function realScreenOptIn({ env = process.env, saved } = {}) {diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjsindex 4c7a45ec..143727e6 100644--- a/apps/server/src/computer-use/shared/portable-host.mjs+++ b/apps/server/src/computer-use/shared/portable-host.mjs@@ -492,7 +492,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =/** One MCP client (an engine connection): JSON-RPC in, JSON-RPC out. *//*** One MCP connection. `internal` false: a connection from outside the app's own engine (the copied CLI- * connection), whose calls belong to no chat of this app: it never keeps or reuses "Allow for this chat".+ * connection), whose calls belong to no chat of this app.*/function connect({ internal = true } = {}) {const connection = { id: randomUUID(), busy: false, closed: false, internal: internal !== false };diff --git a/apps/server/src/computer-use/shared/virtual-host.mjs b/apps/server/src/computer-use/shared/virtual-host.mjsindex 48f87927..35475f6c 100644--- a/apps/server/src/computer-use/shared/virtual-host.mjs+++ b/apps/server/src/computer-use/shared/virtual-host.mjs@@ -17,8 +17,8 @@// (apps/server/src/computer-use/shared/); PARITY.sha256 lists it in both.import { spawn, spawnSync } from "node:child_process";-import { randomUUID } from "node:crypto";-import { existsSync, mkdtempSync, readdirSync, rmSync } from "node:fs";+import { randomBytes, randomUUID } from "node:crypto";+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs";import os from "node:os";import path from "node:path";import { fileURLToPath } from "node:url";@@ -91,66 +91,176 @@ export function browserArgs(browser, { profile, url = "about:blank", width = VIR];}+/** The variables the agent's display and its apps get: an allowlist, never the host's whole environment. */+export const AGENT_ENV_KEEP = Object.freeze(["PATH", "HOME", "USER", "LOGNAME", "LANG", "LANGUAGE", "TZ", "TMPDIR", "SNAP", "SNAP_NAME", "SNAP_USER_COMMON", "FONTCONFIG_PATH", "FONTCONFIG_FILE"]);++/**+ * The agent display's environment. Kept: AGENT_ENV_KEEP and LC_*. Set: the display, its cookie, a private+ * XDG_RUNTIME_DIR (no person's sound server, portals, keyring or single-instance sockets), the session bus off,+ * no portals or accessibility bus. Left out: everything else (SESSION_MANAGER, XDG_CURRENT_DESKTOP, DBUS_*,+ * WAYLAND_DISPLAY, tokens), so nothing the agent starts there reaches the person's desktop.+ */+export function agentEnv(env, { display, xauthority, runtimeDir }) {+ const out = {};+ for (const [key, value] of Object.entries(env ?? {})) if (typeof value === "string" && (AGENT_ENV_KEEP.includes(key) || /^LC_[A-Z_]+$/.test(key))) out[key] = value;+ return {+ ...out,+ DISPLAY: display, XAUTHORITY: xauthority, XDG_SESSION_TYPE: "x11", XDG_RUNTIME_DIR: runtimeDir,+ DBUS_SESSION_BUS_ADDRESS: "disabled:", GTK_USE_PORTAL: "0", NO_AT_BRIDGE: "1", GTK_A11Y: "none",+ PULSE_SERVER: "unix:/nonexistent", PIPEWIRE_REMOTE: "/nonexistent", BROWSER: "/bin/false",+ };+}++/** An Xauthority file with one wildcard MIT-MAGIC-COOKIE-1 entry (any display number): the server's -auth and the clients'. */+export function xauthority(cookie) {+ const field = (buf) => { const len = Buffer.alloc(2); len.writeUInt16BE(buf.length); return Buffer.concat([len, buf]); };+ const family = Buffer.alloc(2); family.writeUInt16BE(0xffff);+ return Buffer.concat([family, field(Buffer.alloc(0)), field(Buffer.alloc(0)), field(Buffer.from("MIT-MAGIC-COOKIE-1")), field(cookie)]);+}+/**- * Linux backend: a new Xvfb display (+ openbox), the virtual worker on it, and, unless `browser` is false, a- * browser on a brand-new profile opened on `url`. Nothing is shared with the person's own browser or display.+ * openbox's configuration on the agent's display: windows open maximized, alt+tab switches them, a title bar+ * moves and closes them. No root menu and no key or mouse binding that runs a program (no terminal there: what the+ * agent runs goes through the engine's shell permissions).+ */+export const OPENBOX_RC = `<?xml version="1.0" encoding="UTF-8"?>+<openbox_config xmlns="http://openbox.org/3.4/rc">+ <focus><focusNew>yes</focusNew><followMouse>no</followMouse></focus>+ <placement><policy>Smart</policy></placement>+ <desktops><number>1</number></desktops>+ <keyboard>+ <keybind key="A-Tab"><action name="NextWindow"/></keybind>+ <keybind key="A-S-Tab"><action name="PreviousWindow"/></keybind>+ </keyboard>+ <mouse>+ <context name="Client"><mousebind button="Left" action="Press"><action name="Focus"/><action name="Raise"/></mousebind></context>+ <context name="Titlebar"><mousebind button="Left" action="Press"><action name="Focus"/><action name="Raise"/></mousebind><mousebind button="Left" action="Drag"><action name="Move"/></mousebind></context>+ <context name="Close"><mousebind button="Left" action="Click"><action name="Close"/></mousebind></context>+ </mouse>+ <menu><file>/dev/null</file></menu>+ <applications><application class="*"><maximized>yes</maximized><decor>yes</decor></application></applications>+</openbox_config>+`;++/**+ * A watchdog for when the host dies without cleaning up (SIGKILL, a crash): a detached /bin/sh that waits for+ * `hostPid` to go, then ends the process groups and removes the folders. Returns { stop() }.+ */+export function startWatchdog({ hostPid = process.pid, groups = [], dirs = [], everyMs = 1000, spawnFn = spawn } = {}) {+ const ids = groups.filter((g) => Number.isSafeInteger(g) && g > 1);+ // Only folders this backend made (omnirush-cu-*), never anything else.+ const paths = dirs.filter((d) => typeof d === "string" && /\/omnirush-cu-[^/'"]+$/.test(d));+ const quote = (s) => `'${String(s).replace(/'/g, "'\\''")}'`;+ const script = `while kill -0 ${Number(hostPid)} 2>/dev/null; do sleep ${Math.max(0.2, everyMs / 1000)}; done; ${ids.map((g) => `kill -TERM -${g} 2>/dev/null;`).join(" ")} sleep 1; ${ids.map((g) => `kill -KILL -${g} 2>/dev/null;`).join(" ")} ${paths.length ? `rm -rf ${paths.map(quote).join(" ")}` : "true"}`;+ const child = spawnFn("/bin/sh", ["-c", script], { stdio: "ignore", detached: true });+ child.on?.("error", () => undefined);+ child.unref?.();+ return { pid: child.pid, stop() { try { process.kill(-child.pid, "SIGKILL"); } catch { try { child.kill?.("SIGKILL"); } catch { /* gone */ } } } };+}++/** Chromium's own words when its sandbox cannot start here (no user namespaces, a SUID helper not set up). */+export const SANDBOX_ERRORS = /No usable sandbox|setuid_sandbox_host|SUID sandbox helper|apparmor-userns-restrictions|Failed to move to new namespace|zygote_host_impl_linux/i;++/**+ * Linux backend: a new Xvfb display (its number from -displayfd, a random cookie), openbox with OPENBOX_RC, the+ * virtual worker on it, and, unless `browser` is false, a browser on a brand-new profile opened on `url`. Every+ * program there gets agentEnv(). Nothing is shared with the person's own browser, display or desktop session.*/export async function startXvfbBackend({ name = "main", env = process.env, nodePath = process.execPath, nodeEnv = {}, koffiPath = null, url = null, browser = true, size = VIRTUAL_SIZE, deps = {} } = {}) {- if ((deps.platform ?? process.platform) !== "linux") return failure("virtual_unavailable", "This display needs Linux with Xvfb.");- if (!which("Xvfb", env)) return failure("virtual_unavailable", "Xvfb is not installed (Debian/Ubuntu: sudo apt install xvfb openbox).");- const number = freeDisplay(90);- if (number === null) return failure("virtual_unavailable", "No free X display number.");- const disp = `:${number}`;+ if ((deps.platform ?? process.platform) !== "linux") return failure("virtual_unavailable", "The agent's own display needs Linux with Xvfb.");+ if (!which("Xvfb", env)) return failure("virtual_unavailable", "The agent's own display needs Xvfb, which is not installed (Debian/Ubuntu: sudo apt install xvfb openbox).");+ const home = mkdtempSync(path.join(os.tmpdir(), "omnirush-cu-"));+ chmodSync(home, 0o700);+ const runtimeDir = path.join(home, "run");+ mkdirSync(runtimeDir, { mode: 0o700 });+ const authFile = path.join(home, "Xauthority");+ writeFileSync(authFile, xauthority(randomBytes(16)), { mode: 0o600 });const children = [];+ let profile = null, watchdog = null, closed = false;const kill = (child) => { if (!child?.pid) return; try { process.kill(-child.pid, "SIGTERM"); } catch { try { child.kill(); } catch { /* gone */ } } };- const xvfb = spawn("Xvfb", [disp, "-screen", "0", `${size.width}x${size.height}x24`, "-nolisten", "tcp"], { stdio: "ignore", env, detached: true });- xvfb.on("error", () => undefined);- children.push(xvfb);- for (let i = 0; i < 60 && !existsSync(`/tmp/.X11-unix/X${number}`); i++) await sleep(100);- if (!existsSync(`/tmp/.X11-unix/X${number}`)) { kill(xvfb); return failure("virtual_unavailable", "Xvfb did not start."); }- const displayEnv = { ...env, DISPLAY: disp, XDG_SESSION_TYPE: "x11", WAYLAND_DISPLAY: "" };+ const removeDirs = () => { for (const dir of [profile, home]) if (dir) { try { rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ } } };+ // The host exiting normally (a thrown error included) takes its display, window manager and browser with it;+ // the watchdog covers a host that is killed.+ const onExit = () => { for (const child of children) kill(child); removeDirs(); };+ process.once("exit", onExit);+ const fail = (code, message) => { process.removeListener("exit", onExit); for (const child of [...children].reverse()) kill(child); removeDirs(); return failure(code, message); };++ // A display number from 90 up (never the usual :0-:10 of a person's desktop). The server says on fd 3 when it is+ // ready; one that exits (another server took the number between the check and the start) is retried higher.+ let xvfb = null, number = null, from = 90;+ for (let attempt = 0; attempt < 8 && number === null; attempt += 1) {+ const candidate = freeDisplay(from);+ if (candidate === null) break;+ from = candidate + 1;+ const child = spawn("Xvfb", [`:${candidate}`, "-displayfd", "3", "-screen", "0", `${size.width}x${size.height}x24`, "-nolisten", "tcp", "-auth", authFile], { stdio: ["ignore", "ignore", "ignore", "pipe"], env: { PATH: env.PATH ?? "/usr/bin:/bin" }, detached: true });+ child.on("error", () => undefined);+ const ready = await new Promise((resolve) => {+ let text = "";+ const timer = setTimeout(() => resolve(null), 10_000);+ child.stdio[3]?.on("data", (d) => { text += d; const m = /^(\d+)\s/.exec(text); if (m) { clearTimeout(timer); resolve(Number(m[1])); } });+ child.once("exit", () => { clearTimeout(timer); resolve(null); });+ });+ if (ready !== null && child.exitCode === null) { xvfb = child; number = ready; children.push(child); }+ else kill(child);+ }+ if (number === null || !xvfb) return fail("virtual_unavailable", "Xvfb did not start.");+ const disp = `:${number}`;+ const displayEnv = agentEnv(env, { display: disp, xauthority: authFile, runtimeDir });let wm = null;if (which("openbox", env)) {- const child = spawn("openbox", ["--sm-disable"], { stdio: "ignore", env: displayEnv, detached: true });+ const rc = path.join(home, "openbox-rc.xml");+ writeFileSync(rc, OPENBOX_RC, { mode: 0o600 });+ const child = spawn("openbox", ["--sm-disable", "--config-file", rc], { stdio: "ignore", env: displayEnv, detached: true });child.on("error", () => undefined);children.push(child); wm = "openbox";}- const worker = deps.worker ? deps.worker(disp) : createNativeWorker({ workerPath: VIRTUAL_WORKER_PATH, nodePath, nodeEnv: { ...nodeEnv, DISPLAY: disp, XDG_SESSION_TYPE: "x11", WAYLAND_DISPLAY: "" }, koffiPath });+ const worker = deps.worker ? deps.worker(disp) : createNativeWorker({ workerPath: VIRTUAL_WORKER_PATH, nodePath, nodeEnv: { ...nodeEnv, DISPLAY: disp, XAUTHORITY: authFile, XDG_SESSION_TYPE: "x11", WAYLAND_DISPLAY: "" }, koffiPath });const ready = await worker.ready;- let profile = null, opened = null;- // The host exiting (a crash included) takes its display, window manager and browser with it.- const onExit = () => { for (const child of children) kill(child); if (profile) { try { rmSync(profile, { recursive: true, force: true }); } catch { /* best effort */ } } };- process.once("exit", onExit);- const close = async () => {- process.removeListener("exit", onExit);- await worker.close?.().catch(() => undefined);- for (const child of [...children].reverse()) kill(child);- if (profile) { await sleep(300); try { rmSync(profile, { recursive: true, force: true }); } catch { /* best effort */ } }- };- if (!ready.supported) { await close(); return failure(ready.code ?? "virtual_unavailable", ready.error ?? "The display's worker did not start."); }+ if (!ready.supported || xvfb.exitCode !== null) { await worker.close?.().catch(() => undefined); return fail(ready.code ?? "virtual_unavailable", ready.error ?? "The display's worker did not start."); }++ let opened = null, warning = null;const found = browser ? findBrowser(env) : null;if (found) {- profile = mkdtempSync(path.join(os.tmpdir(), "omnirush-cu-browser-"));+ // A snap browser cannot read the host's /tmp: its profile lives in the snap's own folder, removed the same way.+ const snap = found.path.startsWith("/snap/") || (() => { try { return realpathSync(found.path).startsWith("/snap/"); } catch { return false; } })();+ const base = snap && env.HOME ? path.join(env.HOME, "snap", found.name.replace(/-browser$/, ""), "common") : os.tmpdir();+ if (snap) mkdirSync(base, { recursive: true });+ profile = mkdtempSync(path.join(base, "omnirush-cu-browser-"));const launch = (noSandbox) => {- const child = spawn(found.path, browserArgs(found, { profile, url: url ?? "about:blank", width: size.width, height: size.height, root: noSandbox }), { stdio: "ignore", env: displayEnv, detached: true });+ const child = spawn(found.path, browserArgs(found, { profile, url: url ?? "about:blank", width: size.width, height: size.height, root: noSandbox }), { stdio: ["ignore", "ignore", "pipe"], env: displayEnv, detached: true });child.on("error", () => undefined);+ let stderr = "";+ child.stderr?.on("data", (d) => { if (stderr.length < 65_536) stderr += d; });+ child.stderr?.resume();children.push(child);- return child;+ return { child, stderr: () => stderr };};- // Chromium's sandbox first. Where it cannot start (root, or Ubuntu 23.10+ without the browser's AppArmor- // profile: no unprivileged user namespaces) the browser exits at once: then without it, as Playwright does,- // and the session record says so.+ // Chromium's sandbox, always, except as root (where it cannot run) or when the browser itself says it cannot+ // start one here (Ubuntu 23.10+ without its AppArmor profile, a SUID helper not set up). Then it runs+ // without, and the person is told (the open's note, the session's status line, the record).const root = typeof process.getuid === "function" && process.getuid() === 0;let sandbox = found.kind === "chromium" && !root;- let child = launch(!sandbox);+ let run = launch(found.kind === "chromium" && root);if (sandbox) {- const exited = await Promise.race([new Promise((resolve) => child.once("exit", () => resolve(true))), sleep(2500).then(() => false)]);- if (exited) { sandbox = false; child = launch(true); }+ const exited = await Promise.race([new Promise((resolve) => run.child.once("exit", () => resolve(true))), sleep(3000).then(() => false)]);+ if (exited && SANDBOX_ERRORS.test(run.stderr())) { sandbox = false; run = launch(true); }}+ if (found.kind === "chromium" && !sandbox) warning = root ? "The agent's browser runs without Chromium's sandbox because OmniRush runs as root." : "The agent's browser runs without Chromium's sandbox: this system does not allow it (no unprivileged user namespaces). Install the browser's AppArmor profile to turn it back on.";opened = { name: found.name, kind: found.kind, version: versionOf(found.path, ["--version"], env), profile: "fresh", ...(found.kind === "chromium" ? { sandbox } : {}) };}+ watchdog = (deps.watchdog ?? startWatchdog)({ hostPid: process.pid, groups: children.map((c) => c.pid), dirs: [home, profile].filter(Boolean) });+ const close = async () => {+ if (closed) return;+ closed = true;+ process.removeListener("exit", onExit);+ await worker.close?.().catch(() => undefined);+ for (const child of [...children].reverse()) kill(child);+ await sleep(300);+ removeDirs();+ watchdog?.stop();+ };const packages = deps.packages ?? virtualPackages(env);- return { ok: true, kind: "xvfb", name, display: disp, width: ready.size?.width ?? size.width, height: ready.size?.height ?? size.height, wm, packages, browser: opened, worker, close };+ return { ok: true, kind: "xvfb", name, display: disp, width: ready.size?.width ?? size.width, height: ready.size?.height ?? size.height, wm, packages, browser: opened, ...(warning ? { warning } : {}), worker, close, pids: children.map((c) => c.pid), dirs: [home, profile].filter(Boolean) };}// ------------------------------------------------------------------- host@@ -194,7 +304,7 @@ export function createVirtualHost({ startBackend, settings = () => ({}), contextconst emit = (s, event, by, extra = {}) => { try { onEvent({ event, by, session_id: s.id, app_id: s.app_id, app_name: s.app_name, bundle_or_exe: s.bundle, mode: s.mode, screen: s.screen, os: osLabel, virtual: s.virtualInfo, ...extra }); } catch { /* trace is best effort */ } };const windowOf = (s) => ({ id: "0", app_id: s.app_id, app_name: s.app_name, bundle_or_exe: s.bundle, title: s.title, origin: { x: 0, y: 0 }, bounds: { x: 0, y: 0, width: s.screen.width, height: s.screen.height } });const publicState = () => [...sessions.values()].map((s) => ({- id: s.id, phase: s.phase, mode: s.mode, purpose: s.purpose, status: "",+ id: s.id, phase: s.phase, mode: s.mode, purpose: s.purpose, status: s.warning ?? "",app: { app_id: s.app_id, app_name: s.app_name, bundle_or_exe: s.bundle, pid: null }, window: { id: "0", title: s.title }, windows: [],approval: "virtual", actions: s.actions, actions_left: Math.max(0, limits().actions_per_session - s.actions), seconds_left: Math.max(0, Math.ceil((s.expiresAt - Date.now()) / 1000)),can_continue: false, log: s.log.slice(-30), frame: s.lastFrame ?? null, chat: s.chat ?? null, hidden: s.hidden === true,@@ -295,10 +405,10 @@ export function createVirtualHost({ startBackend, settings = () => ({}), contexttitle: backend.browser ? `${backend.browser.name} (fresh profile) on ${label}` : label, mode: args.mode, purpose: args.purpose, phase: "working", chat: ctx.chat ?? null,worker: backend.worker, close: backend.close, screen: { id: String(backend.display), width: backend.width, height: backend.height, scale: 1 },virtualInfo: { width: backend.width, height: backend.height, wm: backend.wm ?? null, packages: Object.entries(backend.packages ?? {}).filter(([, v]) => v).map(([k, v]) => `${k} ${v}`), ...(backend.browser ? { browser: `${backend.browser.name}${backend.browser.version ? ` ${backend.browser.version}` : ""}`, ...(backend.browser.sandbox === false ? { browser_sandbox: false } : {}) } : {}), ...(backend.kind !== "xvfb" ? { kind: backend.kind } : {}) },- actions: 0, requests: new Set(), observation: null, log: [], expiresAt: Date.now() + l.session_seconds * 1000, lastAct: 0, cursor: null,+ actions: 0, requests: new Set(), observation: null, log: [], expiresAt: Date.now() + l.session_seconds * 1000, lastAct: 0, cursor: null, warning: backend.warning ?? null,};sessions.set(s.id, s);- emit(s, "opened", "agent");+ emit(s, "opened", "agent", backend.warning ? { warning: backend.warning } : {});emit(s, "approved", "agent");// The fresh browser comes up: the first observation shows it, not an empty screen.if (backend.browser) { await waitForWindow(s, 15_000).catch(() => null); await settle(s, 4000).catch(() => null); }@@ -308,8 +418,9 @@ export function createVirtualHost({ startBackend, settings = () => ({}), contextexpires_in_seconds: l.session_seconds, actions_left: l.actions_per_session,...(backend.browser ? { browser: { name: backend.browser.name, profile: "fresh" } } : {}),note: backend.kind === "xvfb"- ? `This display (DISPLAY=${s.display}) is yours alone: its own cursor${backend.browser ? ` and a ${backend.browser.name} window on a fresh profile` : ""}. Nothing here needs approval. Use ctrl+t / ctrl+tab / ctrl+w for tabs and ctrl+l for the address bar; start other apps yourself with DISPLAY=${s.display} <app> &, then observe.`- : "This browser is yours alone: its own cursor and a fresh profile. Nothing here needs approval. Use ctrl+t / ctrl+tab / ctrl+w for tabs and ctrl+l for the address bar, then observe.",+ ? `This display is yours alone: its own cursor${backend.browser ? ` and a ${backend.browser.name} window on a fresh profile` : ""}. Nothing here needs approval. Use ctrl+t / ctrl+tab / ctrl+w for tabs, ctrl+l for the address bar and alt+tab for windows, then observe.${backend.warning ? ` Note: ${backend.warning}` : ""}`+ : `This browser is yours alone: its own cursor and a fresh profile. Nothing here needs approval. Use ctrl+t / ctrl+tab / ctrl+w for tabs and ctrl+l for the address bar, then observe.${backend.warning ? ` Note: ${backend.warning}` : ""}`,+ ...(backend.warning ? { warning: backend.warning } : {}),};}diff --git a/apps/server/src/computer-use/shared/virtual-worker.mjs b/apps/server/src/computer-use/shared/virtual-worker.mjsindex 902ec10e..e731038f 100644--- a/apps/server/src/computer-use/shared/virtual-worker.mjs+++ b/apps/server/src/computer-use/shared/virtual-worker.mjs@@ -64,7 +64,9 @@ function open() {const queryPointer = x.func("int XQueryPointer(void *display, ulong window, _Out_ ulong *root, _Out_ ulong *child, _Out_ int *root_x, _Out_ int *root_y, _Out_ int *win_x, _Out_ int *win_y, _Out_ uint *mask)");const downKeys = new Set();const downButtons = new Set();- let spare = null;+ let spares = null; // keycodes with no symbols, lent to characters the layout does not have+ const lent = new Map(); // keysym -> keycode currently mapped to it+ let nextSpare = 0;const size = () => { const a = {}; if (!getAttributes(display, root, a)) fail("desktop_unavailable", "The virtual display is gone."); return { width: a.width, height: a.height }; };const pointer = () => { const r = [0], c = [0], rx = [0], ry = [0], wx = [0], wy = [0], m = [0]; queryPointer(display, root, r, c, rx, ry, wx, wy, m); return { x: rx[0], y: ry[0] }; };const held = () => send({ event: "held", keys: [...downKeys], buttons: [...downButtons] });@@ -87,24 +89,28 @@ function open() {sync(display, 0);};const codeOf = (symbol) => keycode(display, typeof symbol === "number" ? symbol : keysym(symbol));- // A keycode with no symbols, borrowed to type a character the layout does not have (as xdotool does).- const spareCode = () => {- if (spare !== null) return spare;+ // Keycodes with no symbols, lent to characters the layout does not have (as xdotool does). A pool, used in turn+ // and kept mapped until reused, so an app reading the mapping late still sees the right character; all restored+ // when the worker ends.+ const spareCodes = () => {+ if (spares) return spares;+ spares = [];const min = [0], max = [0]; displayKeycodes(display, min, max);const per = [0], ptr = getMapping(display, min[0], max[0] - min[0] + 1, per);- if (!ptr) return (spare = 0);+ if (!ptr) return spares;try {const syms = koffi.decode(ptr, "ulong", (max[0] - min[0] + 1) * per[0]);- for (let code = max[0]; code >= min[0]; code -= 1) {+ for (let code = max[0]; code >= min[0] && spares.length < 16; code -= 1) {const row = Array.from(syms.slice((code - min[0]) * per[0], (code - min[0] + 1) * per[0]), Number);- if (row.every((s) => s === 0)) return (spare = code);+ if (row.every((v) => v === 0)) spares.push(code);}} finally { free(ptr); }- return (spare = 0);+ return spares;};const remap = (code, symbol) => { changeMapping(display, code, 1, [symbol], 1); sync(display, 0); };- // One character: its own key (Shift when it sits on level 2), else the spare keycode remapped to it,- // so any Unicode text arrives as the character itself (no IME composition involved).+ const restoreSpares = () => { for (const code of lent.values()) remap(code, 0); lent.clear(); };+ // One character: its own key (Shift when it sits on level 2), else a lent keycode mapped to it, so any Unicode+ // text arrives as the character itself (no IME composition involved).async function typeSymbol(symbol) {const code = codeOf(symbol);const level0 = code ? Number(symbolAt(display, code, 0)) : 0, level1 = code ? Number(symbolAt(display, code, 1)) : 0;@@ -116,11 +122,18 @@ function open() {if (shifted) press(shift, false);return;}- const borrowed = spareCode();- if (!borrowed) fail("unsupported_action", "This character cannot be typed in the virtual display.");- remap(borrowed, symbol);- await wait(12);- try { press(borrowed, true); press(borrowed, false); await wait(8); } finally { remap(borrowed, 0); }+ let borrowed = lent.get(symbol);+ if (!borrowed) {+ const pool = spareCodes();+ if (!pool.length) fail("unsupported_action", "This character cannot be typed in the virtual display.");+ borrowed = pool[nextSpare++ % pool.length];+ for (const [sym, c] of lent) if (c === borrowed) lent.delete(sym);+ remap(borrowed, symbol);+ lent.set(symbol, borrowed);+ await wait(15);+ }+ press(borrowed, true); press(borrowed, false);+ await wait(4);}const NAMES = { Enter: "Return", Tab: "Tab", Escape: "Escape", Backspace: "BackSpace", Delete: "Delete", Insert: "Insert", Space: "space", Left: "Left", Up: "Up", Right: "Right", Down: "Down", Home: "Home", End: "End", PageUp: "Prior", PageDown: "Next", CapsLock: "Caps_Lock", ContextMenu: "Menu", PrintScreen: "Print" };const MODS = { ctrl: "Control_L", shift: "Shift_L", alt: "Alt_L", meta: "Super_L" };@@ -221,11 +234,16 @@ function open() {for (const b of new Set([1, 2, 3, ...buttons.filter((n) => Number.isInteger(n) && n >= 1 && n <= 9)])) rawButton(display, b, 0, 0);sync(display, 0);}- return { size, capture, act, pointer, release, up, close() { release(); close(display); } };+ return { size, capture, act, pointer, release, up, close() { release(); restoreSpares(); close(display); } };}// Worker mode only when run as the process's script (importing it, as tests do, runs nothing).const isMain = (() => { try { return Boolean(process.argv[1]) && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); } catch { return false; } })();+// `--release <json>`: a one-shot run after the worker died with input held (portable-host createNativeWorker).+if (isMain && process.argv[2] === "--release") {+ try { const b = open(); b.up(JSON.parse(process.argv[3] || "{}")); b.close(); } catch { /* the display is gone */ }+ process.exit(0);+}if (isMain && process.send) {let backend = null;try {diff --git a/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts b/apps/server/src/opencode-plugins/omnirush-mcp-policy.tsindex 44c73fa2..af1c48e5 100644--- a/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts+++ b/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts@@ -22,7 +22,7 @@ import { readWorkspaceMcp } from "../mcp-allow/state.js";import { computerUseBuiltins, COMPUTER_USE_DIR_ENV } from "../computer-use/desktop.js";import { isComputerTool, SERVER_ID } from "../computer-use/shared/contract.mjs";import { pruneImages } from "../computer-use/shared/media.mjs";-import { SCREEN_BASH_RULES, subagentPermissionRules } from "../computer-use/shared/policy.mjs";+import { REAL_SCREEN_DENY_RULES, SCREEN_BASH_RULES, subagentPermissionRules } from "../computer-use/shared/policy.mjs";import { boxHome, desktopProjects, keyOf, record } from "../box/plugin-lib.js";function isRecord(value: unknown): value is Record<string, unknown> {@@ -64,6 +64,30 @@ function withScreenBashRules(permission: Record<string, unknown>): Record<stringreturn { ...permission, bash: next };}+/**+ * Every mode, computer use on or off: the agent can neither run a command that turns the person's real screen on+ * nor write that setting (policy.mjs REAL_SCREEN_DENY_RULES), appended after the mode's own rules (last match wins).+ */+export function withRealScreenDeny(value: unknown): Record<string, unknown> {+ const permission: Record<string, unknown> = typeof value === "string" ? { "*": value } : isRecord(value) ? { ...value } : {};+ for (const [key, denies] of Object.entries(REAL_SCREEN_DENY_RULES)) {+ // Only the deny patterns are added: every other command or file keeps the rules it had.+ const current = permission[key];+ permission[key] = { ...(isRecord(current) ? current : typeof current === "string" ? { "*": current } : {}), ...denies };+ }+ return permission;+}++function applyRealScreenDeny(target: Record<string, unknown>): void {+ target.permission = withRealScreenDeny(target.permission);+ if (!isRecord(target.agent)) return;+ const agents: Record<string, unknown> = { ...target.agent };+ for (const [name, agent] of Object.entries(agents)) {+ if (isRecord(agent) && agent.permission !== undefined) agents[name] = { ...agent, permission: withRealScreenDeny(agent.permission) };+ }+ target.agent = agents;+}+/** Adds the left-out `computer-use` entry to the project's MCP report (Settings > MCP). */function reportReserved(root: string, names: string[]): void {const dir = process.env.OMNIRUSH_MCP_REPORT_DIR;@@ -151,7 +175,7 @@ export default async function omnirushMcpPolicy(input?: { directory?: string; clconst envOnly = (target as Record<symbol, unknown>)[MCP_ENV_VALUES_MARK] === true;const environment = { ...mcpServerEnvironment(process.env, {}, [], process.platform, { envOnly }), ...own };target.mcp = { ...(isRecord(target.mcp) ? target.mcp : {}), [SERVER_ID]: { ...builtin, environment } };- // The host asks the person itself (never the engine's permission prompts), in every mode.+ // Computer use never asks, so the engine never asks for its tools either, in every mode.const permission: Record<string, unknown> = typeof target.permission === "string" ? { "*": target.permission } : isRecord(target.permission) ? { ...target.permission } : {};delete permission[`${SERVER_ID}_*`];target.permission = withScreenBashRules({ ...permission, [`${SERVER_ID}_*`]: "allow" });@@ -160,6 +184,7 @@ export default async function omnirushMcpPolicy(input?: { directory?: string; cl}withoutSubagentComputerTools(target);}+ if (target) applyRealScreenDeny(target);},// Old screenshots of computer tools leave the model's context in blocks (keep 12, drop 8);// compaction requests serialize messages to text and carry no images anyway.