Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

feat: real screen only with the account's server-held opt-in

ogsamratf1da377c4dc0
diff --git a/apps/app/src/react-app/domains/settings/computer-use-config.tsx b/apps/app/src/react-app/domains/settings/computer-use-config.tsx
index 892ead28..ed9fa13e 100644
--- a/apps/app/src/react-app/domains/settings/computer-use-config.tsx
+++ b/apps/app/src/react-app/domains/settings/computer-use-config.tsx
@@ -168,14 +168,15 @@ export function ComputerUseConfig({ connected, connecting, onConnect, onRefresh,
<div>
<p className="text-sm font-medium">Use my own screen</p>
<p className="mt-1 text-xs leading-relaxed text-muted-foreground" id="computer-use-real-screen-help">
- {realScreen.data?.allowed === false ? "Turned off for your account." : realScreen.data?.forced ? "Set by the OMNIRUSH_COMPUTER_USE_REAL_SCREEN environment variable." : "Off by default. When on, the agent sees and uses the apps on your screen without asking. Your own mouse or keyboard pauses it until you choose Continue."}
+ {realScreen.data?.forced ? "Set by the OMNIRUSH_COMPUTER_USE_REAL_SCREEN environment variable." : "Off by default. When on, the agent sees and uses the apps on your screen without asking. Your own mouse or keyboard pauses it until you choose Continue."}
+ {realScreen.data && !realScreen.data.allowed ? " It also needs your account's yes: turn on \u201cLet the agent use my real screen\u201d in your account at omnirush.ai/console/account. The app cannot turn that on; until then the agent uses its own display." : null}
</p>
</div>
<Switch
aria-label="Use my own screen"
aria-describedby="computer-use-real-screen-help"
checked={realScreenOn}
- disabled={!supported || !realScreen.data || realScreen.data.forced || realScreen.data.allowed === false || setRealScreen.isPending}
+ disabled={!supported || !realScreen.data || realScreen.data.forced || setRealScreen.isPending}
onCheckedChange={(on: boolean) => setRealScreen.mutate(on)}
data-computer-use-real-screen
/>
diff --git a/apps/desktop/electron/computer-use.mjs b/apps/desktop/electron/computer-use.mjs
index 20ca7bcd..3b042f1a 100644
--- a/apps/desktop/electron/computer-use.mjs
+++ b/apps/desktop/electron/computer-use.mjs
@@ -76,6 +76,10 @@ let failedAt = 0; // a host that did not start is tried again after a minute
const cursors = new Map(); // session id → the agent cursor's latest move (live view)
let cursorSeq = 0;
let realScreenNoticeShown = false;
+// The server's yes for this account's real screen (`real_screen` and `real_screen_opt_in`), taken only from a
+// fresh read (the refresh route answers from the server's answer in memory). The 5 s state poll reads the local
+// state folder, which the agent's shell can write: it can only take this back to no, never to yes.
+let serverRealScreen = false;
function shippedModule(name) {
// Packaged: app.asar/server/dist; dev: apps/server/dist (OMNIRUSH_DEV_MODE prefers it).
@@ -155,9 +159,21 @@ function realScreenOn() {
return policy ? policy.realScreenOptIn({ env: process.env, saved: savedRealScreen(policy), allowEnvOn: !app.isPackaged }) : false;
}
+/** Whether a state answer says yes for this account's real screen (both keys exactly true). */
+function saysRealScreen(answer) {
+ return Boolean(shared?.policyModule?.serverRealScreen?.(answer?.setting)) && answer?.enabled === true;
+}
+
+/** Ends the sessions on the person's screen (the agent's own display keeps running). */
+async function stopRealScreen() {
+ if (!host) return;
+ if (typeof host.stopRealScreen === "function") await host.stopRealScreen("server").catch(() => {});
+ else for (const s of host.state()) if (s.display !== "virtual" && s.phase !== "ended") await host.control({ session_id: s.id, action: "stop" }).catch(() => {});
+}
+
function hostSettings() {
const setting = server?.setting ?? {};
- return { enabled: server?.enabled === true, real_screen: setting.real_screen !== false, real_screen_on: realScreenOn(), virtual_display: setting.virtual_display !== false, limits: setting.limits ?? null, deny_apps: setting.deny_apps ?? [], message: setting.message ?? null };
+ return { enabled: server?.enabled === true, real_screen: serverRealScreen, real_screen_opt_in: serverRealScreen, real_screen_on: realScreenOn(), virtual_display: setting.virtual_display !== false, limits: setting.limits ?? null, deny_apps: setting.deny_apps ?? [], message: setting.message ?? null };
}
async function hostContext() {
@@ -172,8 +188,10 @@ async function hostContext() {
/** Before each open: the server's switch and limits read now (not a cached answer). False when unread. */
async function hostRefresh() {
const fresh = await serverFetch("/computer-use/refresh", { method: "POST", body: JSON.stringify({ max_age_ms: 0 }) }).catch(() => null);
- if (!fresh) return false;
+ if (!fresh) { serverRealScreen = false; return false; }
server = fresh;
+ // Only the server's answer just read can say yes; anything else is no.
+ serverRealScreen = fresh.read === true && saysRealScreen(fresh);
return fresh.read === true;
}
@@ -215,7 +233,7 @@ async function startHost() {
preview.bind(created);
host = created;
// The real screen on: said once per launch (the setting is a file the agent's shell could also write).
- if (!realScreenNoticeShown && realScreenOn() && server?.setting?.real_screen !== false) {
+ if (!realScreenNoticeShown && realScreenOn() && serverRealScreen) {
realScreenNoticeShown = true;
try { if (Notification.isSupported()) new Notification({ title: "Computer use can use your own screen", body: "The agent can click and type in your apps without asking. Stop it any time with Stop; turn it off in Settings > Computer use." }).show(); } catch { /* best effort */ }
}
@@ -245,6 +263,10 @@ async function stopHost(by = "server") {
async function apply(next) {
server = next;
+ // A periodic answer can only take the account's real screen away; a fresh read gives it (hostRefresh).
+ if (next?.read === true) serverRealScreen = saysRealScreen(next);
+ else if (!saysRealScreen(next)) serverRealScreen = false;
+ if (!serverRealScreen) await stopRealScreen();
if (server?.enabled && supported() && !hostPromise && Date.now() - failedAt < 60_000) { pushStatus(); return; }
if (server?.enabled && supported()) await startHost().catch((error) => { failedAt = Date.now(); deps?.log?.("computer use host did not start", error); });
else if (host) await stopHost("server");
@@ -256,7 +278,8 @@ function scheduleRefresh() {
const open = host?.state().some((s) => s.phase !== "ended") ?? false;
if (open && !refreshTimer) {
refreshTimer = setInterval(() => {
- void serverFetch("/computer-use/refresh", { method: "POST", body: "{}" }).then(apply).catch(() => {});
+ // A failed read is a no for the person's screen (fail closed); the agent's own display keeps running.
+ void serverFetch("/computer-use/refresh", { method: "POST", body: "{}" }).then(apply).catch(() => { serverRealScreen = false; void stopRealScreen(); });
}, 60_000);
refreshTimer.unref?.();
} else if (!open && refreshTimer) {
@@ -285,8 +308,9 @@ async function getComputerUseRealScreen() {
const { policyModule } = await sharedModules();
// forced: OMNIRUSH_COMPUTER_USE_REAL_SCREEN decides, whatever is saved.
const forced = policyModule.realScreenOptIn({ env: process.env, saved: true, allowEnvOn: !app.isPackaged }) === policyModule.realScreenOptIn({ env: process.env, saved: false, allowEnvOn: !app.isPackaged });
- // allowed: the server lets the person's screen be used at all.
- return { on: realScreenOn(), forced, allowed: server?.setting?.real_screen !== false };
+ // allowed: the server says yes for this account (its opt-in, set only in the web console), read just now.
+ await hostRefresh().catch(() => false);
+ return { on: realScreenOn(), forced, allowed: serverRealScreen };
}
async function setComputerUseRealScreen(on) {
diff --git a/apps/server/src/computer-use/contract-policy.test.ts b/apps/server/src/computer-use/contract-policy.test.ts
index 9ba5cfcc..a8ca27d8 100644
--- a/apps/server/src/computer-use/contract-policy.test.ts
+++ b/apps/server/src/computer-use/contract-policy.test.ts
@@ -139,7 +139,7 @@ describe("contract v2: validation", () => {
});
describe("policy: server switch", () => {
- const block = { enabled: true, clients: ["desktop"], os: ["linux", "windows"], models: ["gpt-6-astra"], real_screen: true, virtual_display: false, deny_apps: ["gimp", { match: "steam*", reason: "No games." }], limits: { actions_per_session: 50 }, message: null };
+ const block = { enabled: true, clients: ["desktop"], os: ["linux", "windows"], models: ["gpt-6-astra"], real_screen: true, real_screen_opt_in: true, virtual_display: false, deny_apps: ["gimp", { match: "steam*", reason: "No games." }], limits: { actions_per_session: 50 }, message: null };
test("off without a block, off by the server, env override", () => {
expect(computerUseSetting({ env: {}, block: undefined, client: "desktop" })).toMatchObject({ enabled: false, source: "default" });
diff --git a/apps/server/src/computer-use/desktop.test.ts b/apps/server/src/computer-use/desktop.test.ts
index 46d5cae0..aac07063 100644
--- a/apps/server/src/computer-use/desktop.test.ts
+++ b/apps/server/src/computer-use/desktop.test.ts
@@ -125,6 +125,46 @@ describe("routes: the chat in use, and the policy read before an open", () => {
});
});
+describe("routes: the account's real-screen opt-in comes from the server's answer only", () => {
+ test("refresh answers with the block just read (server yes/no), never a block written into the state folder; a failed read is not a yes", async () => {
+ const routes: Route[] = [];
+ const state = tmp();
+ let block: Record<string, unknown> | null = { enabled: true, real_screen: true, real_screen_opt_in: false };
+ let failRead = false;
+ registerComputerUseRoutes({ routes, stateDir: () => state, jsonResponse: (d, status = 200) => new Response(JSON.stringify(d), { status }), readJsonBody: async (r) => r.json(), approvalMode: async () => "guarded", readCapabilities: async () => { if (failRead) throw new Error("offline"); return { computer_use: block }; }, applyEngine: async () => undefined, chats: createChatTracker(), runningSessions: async () => null, recordTrace: () => undefined });
+ const hit = async (method: string, p: string, body?: unknown) => {
+ const route = routes.find((r) => r.method === method && r.regex.test(p));
+ if (!route) throw new Error("no route");
+ const response = await route.handler({ request: new Request(`http://x${p}`, { method, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }), url: new URL(`http://x${p}`), params: {} } as never);
+ return (await response.json()) as Record<string, unknown> & { setting: Record<string, unknown> };
+ };
+ // The account never opted in: no, whatever the local switch.
+ let read = await hit("POST", "/computer-use/refresh", { max_age_ms: 0 });
+ expect(read.read).toBe(true);
+ expect(read.setting).toMatchObject({ real_screen: false, real_screen_opt_in: false });
+ // An older server (no real_screen_opt_in): no.
+ block = { enabled: true, real_screen: true };
+ read = await hit("POST", "/computer-use/refresh", { max_age_ms: 0 });
+ expect(read.setting).toMatchObject({ real_screen: false, real_screen_opt_in: false });
+ // The account opted in from the web console: yes.
+ block = { enabled: true, real_screen: true, real_screen_opt_in: true };
+ read = await hit("POST", "/computer-use/refresh", { max_age_ms: 0 });
+ expect(read.setting).toMatchObject({ real_screen: true, real_screen_opt_in: true });
+ // Turned off on the server: the next read says no.
+ block = { enabled: true, real_screen: true, real_screen_opt_in: false };
+ // Anything writing a yes into the state folder (the agent's shell can) changes nothing about the fresh answer.
+ writeFileSync(path.join(computerUseDir(state), "server.json"), JSON.stringify({ block: { enabled: true, real_screen: true, real_screen_opt_in: true }, at: Date.now() }));
+ read = await hit("POST", "/computer-use/refresh", { max_age_ms: 0 });
+ expect(read.read).toBe(true);
+ expect(read.setting).toMatchObject({ real_screen: false, real_screen_opt_in: false });
+ // A read that fails is no yes (read false: Electron main and the host refuse the person's screen).
+ writeFileSync(path.join(computerUseDir(state), "server.json"), JSON.stringify({ block: { enabled: true, real_screen: true, real_screen_opt_in: true }, at: Date.now() }));
+ failRead = true;
+ read = await hit("POST", "/computer-use/refresh", { max_age_ms: 0 });
+ expect(read.read).toBe(false);
+ });
+});
+
describe("engine MCP plugin", () => {
async function run(env: Record<string, string>, config: Record<string, unknown>) {
const saved = { ...process.env };
diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.ts
index be9b66d9..6d2af676 100644
--- a/apps/server/src/computer-use/portable-host.test.ts
+++ b/apps/server/src/computer-use/portable-host.test.ts
@@ -92,19 +92,23 @@ function fakeVirtualBackend({ width = 320, height = 200 } = {}) {
return { acts, backend, get closed() { return closed; }, get released() { return released; } };
}
-/** Real-screen settings as Electron main gives them: the server's real_screen and virtual_display, the person's real_screen_on. */
-function rig(options: { limits?: Record<string, number>; enabled?: boolean; chats?: Array<string | null>; realScreenOn?: boolean; realScreen?: boolean; native?: ReturnType<typeof fakeNative> } = {}) {
+/**
+ * Real-screen settings as Electron main gives them: the server's real_screen and real_screen_opt_in (the account's
+ * opt-in, both true unless `realScreen` says no) and virtual_display, the person's real_screen_on; `refresh` is the
+ * server read before each open and discover (true: it answered).
+ */
+function rig(options: { limits?: Record<string, number>; enabled?: boolean; chats?: Array<string | null>; realScreenOn?: boolean; realScreen?: boolean; native?: ReturnType<typeof fakeNative>; refresh?: () => boolean | Promise<boolean> } = {}) {
const native = options.native ?? fakeNative();
const dir = mkdtempSync(path.join(os.tmpdir(), "cu-host-"));
dirs.push(dir);
const events: Array<Record<string, unknown>> = [];
const cursors: CursorMove[] = [];
const display = fakeVirtualBackend();
- const settings: Record<string, unknown> & { enabled: boolean; limits: Record<string, number> } = { enabled: options.enabled ?? true, real_screen: options.realScreen ?? true, virtual_display: true, limits: options.limits ?? {}, deny_apps: [], ...(options.realScreenOn === false ? {} : { real_screen_on: true }) };
+ const settings: Record<string, unknown> & { enabled: boolean; limits: Record<string, number> } = { enabled: options.enabled ?? true, real_screen: options.realScreen ?? true, real_screen_opt_in: options.realScreen ?? true, virtual_display: true, limits: options.limits ?? {}, deny_apps: [], ...(options.realScreenOn === false ? {} : { real_screen_on: true }) };
const context = () => ({ ancestors: [], chat: options.chats ? (options.chats.length > 1 ? options.chats.shift()! : options.chats[0]!) : "ses_1" });
const onEvent = (event: Record<string, unknown>) => events.push(event);
const virtual = createVirtualHost({ startBackend: async () => display.backend, settings: () => settings, context, onEvent, onCursor: (move: CursorMove) => cursors.push(move) });
- const hostPromise = createPortableHost({ native, spool: createMediaSpool(path.join(dir, "spool")), settings: () => settings, context, virtual, onEvent });
+ const hostPromise = createPortableHost({ native, spool: createMediaSpool(path.join(dir, "spool")), settings: () => settings, context, refresh: options.refresh ?? (() => true), virtual, onEvent });
return { native, hostPromise, events, settings, dir, display, cursors };
}
@@ -328,6 +332,68 @@ describe("portable host: the agent's own display (virtual:main)", () => {
});
});
+describe("portable host: the account's real-screen opt-in is the server's", () => {
+ test("server no + this computer on: refused, nothing listed or started", async () => {
+ for (const server of [{ real_screen: false, real_screen_opt_in: false }, { real_screen: true, real_screen_opt_in: false }, { real_screen: true, real_screen_opt_in: undefined }, { real_screen: false, real_screen_opt_in: true }]) {
+ const r = rig();
+ Object.assign(r.settings, server);
+ const host = await r.hostPromise;
+ const c = host.connect();
+ const found = await call(c, "computer_discover", {});
+ expect(found.state).toMatchObject({ ok: true, real_screen: false, apps: [] });
+ expect((await call(c, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "Write" })).state).toMatchObject({ ok: false, code: "real_screen_unavailable" });
+ expect(r.native.calls.some((x) => ["list", "start", "focus"].includes(x.method))).toBe(false);
+ // The agent's own display still opens.
+ expect((await call(c, "computer_open_session", { app_id: "virtual:main", mode: "control", purpose: "Write" })).state).toMatchObject({ ok: true });
+ await host.close();
+ }
+ });
+
+ test("server yes + this computer on: opens with no question", async () => {
+ const r = rig();
+ const host = await r.hostPromise;
+ const c = host.connect();
+ expect((await call(c, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "Write" })).state).toMatchObject({ ok: true, approval: "setting" });
+ await host.close();
+ });
+
+ test("the server could not be read just now (or no reader): refused, even with a yes from before", async () => {
+ for (const refresh of [() => false, () => { throw new Error("offline"); }, async () => { throw new Error("offline"); }]) {
+ const r = rig({ refresh });
+ const host = await r.hostPromise;
+ const c = host.connect();
+ const found = await call(c, "computer_discover", {});
+ expect(found.state).toMatchObject({ real_screen: false, apps: [] });
+ expect((await call(c, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "Write" })).state).toMatchObject({ ok: false, code: "real_screen_unverified" });
+ expect(r.native.calls.some((x) => ["list", "start"].includes(x.method))).toBe(false);
+ await host.close();
+ }
+ const native = fakeNative();
+ const host = await createPortableHost({ native, settings: () => ({ enabled: true, real_screen: true, real_screen_opt_in: true, real_screen_on: true }), context: () => ({ chat: "ses_n" }) });
+ expect((await call(host.connect(), "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "x" })).state).toMatchObject({ ok: false, code: "real_screen_unverified" });
+ await host.close();
+ });
+
+ test("turned off on the server while a session is open: the next call ends it; stopRealScreen leaves the agent's display", async () => {
+ const r = rig();
+ const { host, c, session } = await opened(r);
+ const agent = await call(c, "computer_open_session", { app_id: "virtual:main", mode: "control", purpose: "Write" });
+ expect(agent.state).toMatchObject({ ok: true });
+ r.settings.real_screen_opt_in = false;
+ expect((await call(c, "computer_observe", { session_id: session })).state).toMatchObject({ ok: false, code: "real_screen_unavailable" });
+ expect(r.events.some((e) => e.event === "stopped" && e.by === "server")).toBe(true);
+ expect((await call(c, "computer_observe", { session_id: String(agent.state.session_id) })).state).toMatchObject({ ok: true });
+ // stopRealScreen (Electron main on a no) ends only the person's screen.
+ r.settings.real_screen_opt_in = true;
+ const again = await call(c, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "Write" });
+ expect(again.state).toMatchObject({ ok: true });
+ await host.stopRealScreen("server");
+ expect(host.state().filter((x) => x.display !== "virtual" && x.phase !== "ended")).toEqual([]);
+ expect(host.state().some((x) => x.display === "virtual" && x.phase !== "ended")).toBe(true);
+ await host.close();
+ });
+});
+
describe("portable host: a worker that died", () => {
test("a dead worker: sessions stop; the next open starts a new one (noted on the event); at most 3 in 10 minutes", async () => {
const workers: Array<ReturnType<typeof fakeNative>> = [];
@@ -336,7 +402,8 @@ describe("portable host: a worker that died", () => {
const host = await createPortableHost({
native: first,
nativeFactory: () => { const w = fakeNative(); workers.push(w); return w; },
- settings: () => ({ enabled: true, real_screen: true, real_screen_on: true }),
+ settings: () => ({ enabled: true, real_screen: true, real_screen_opt_in: true, real_screen_on: true }),
+ refresh: () => true,
context: () => ({ chat: "ses_r" }),
onEvent: (event: Record<string, unknown>) => events.push(event),
});
@@ -393,7 +460,7 @@ describe("portable host on Windows: the agent's ancestors come from the worker",
if (method === "ancestors") { asked += 1; return [{ pid: 690, name: "cmd", exe: "C:\\Windows\\System32\\cmd.exe" }, { pid: 700, name: "Code", exe: "C:\\Program Files\\Microsoft VS Code\\Code.exe" }]; }
return call0(method, params);
};
- const host = await createPortableHost({ native, platform: "win32", settings: () => ({ enabled: true, real_screen: true, real_screen_on: true }), context: () => ({ chat: "ses_w" }) });
+ const host = await createPortableHost({ native, platform: "win32", settings: () => ({ enabled: true, real_screen: true, real_screen_opt_in: true, real_screen_on: true }), refresh: () => true, context: () => ({ chat: "ses_w" }) });
const c = host.connect();
const found = await call(c, "computer_discover", {});
expect((found.state.apps as Array<{ name: string; available?: boolean }>).find((a) => a.name === "Code")).toMatchObject({ available: false });
diff --git a/apps/server/src/computer-use/routes.ts b/apps/server/src/computer-use/routes.ts
index 92bfd599..c26c4c40 100644
--- a/apps/server/src/computer-use/routes.ts
+++ b/apps/server/src/computer-use/routes.ts
@@ -4,11 +4,16 @@
* writes when its host listens, and the `computer.session` events it reports.
*
* GET /computer-use/state { enabled, setting, mode, dir, spool, chat }
- * POST /computer-use/refresh reads /collect/capabilities again (≤ 60 s old), then the state
+ * POST /computer-use/refresh reads /collect/capabilities again (≤ 60 s old), then the state; with `read`
+ * true, `setting` is the answer just read (held in memory, never re-read from the
+ * state folder, which the agent's shell can write), so the person's real screen
+ * (`real_screen` and `real_screen_opt_in`, the account's server-held opt-in) can
+ * only come from the server
* POST /computer-use/host { command, environment } | { host: null }
* POST /computer-use/event a computer.session event → the chat's trace
*/
import { addRoute, type Route } from "../routes/registry.js";
+import { computerUseSetting } from "./shared/policy.mjs";
import {
computerUseDir,
computerUseSettingFor,
@@ -44,17 +49,24 @@ export type ComputerUseRouteDeps = {
const REFRESH_FLOOR_MS = 2_000;
+function isObject(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
export function registerComputerUseRoutes(deps: ComputerUseRouteDeps): void {
const { routes, jsonResponse } = deps;
- const state = async (read?: boolean) => {
+ const state = async (read?: boolean, fresh?: unknown) => {
const dir = computerUseDir(deps.stateDir());
- const setting = computerUseSettingFor(dir);
+ // A fresh answer: the block it carries (an answer without one is off); else the block as last remembered.
+ const setting = fresh !== undefined
+ ? computerUseSetting({ env: process.env, block: isObject(fresh) && isObject(fresh.computer_use) ? fresh.computer_use : undefined, client: "desktop" })
+ : computerUseSettingFor(dir);
const mode = permissionModeName(await deps.approvalMode().catch(() => null));
// The chat "Allow for this chat" is kept for: the one prompted chat that runs, else none (asked every time).
const chat = deps.chats.only(await deps.runningSessions().catch(() => null));
return {
enabled: setting.enabled,
- setting: { enabled: setting.enabled, real_screen: setting.real_screen, virtual_display: setting.virtual_display, limits: setting.limits, deny_apps: setting.deny_apps, models: setting.models, message: setting.message, ask_send_delete: setting.ask_send_delete === true, reason: setting.reason, source: setting.source },
+ setting: { enabled: setting.enabled, real_screen: setting.real_screen, real_screen_opt_in: setting.real_screen_opt_in, virtual_display: setting.virtual_display, limits: setting.limits, deny_apps: setting.deny_apps, models: setting.models, message: setting.message, ask_send_delete: setting.ask_send_delete === true, reason: setting.reason, source: setting.source },
mode,
dir,
spool: mediaSpoolDir(deps.stateDir()),
@@ -74,14 +86,16 @@ export function registerComputerUseRoutes(deps: ComputerUseRouteDeps): void {
// that recent counts as read now.
const maxAge = typeof body.max_age_ms === "number" && Number.isFinite(body.max_age_ms) ? Math.min(60_000, Math.max(REFRESH_FLOOR_MS, Math.round(body.max_age_ms))) : 60_000;
let read = false;
+ let fresh: unknown;
try {
const capabilities = await deps.readCapabilities(maxAge);
if (rememberComputerUseSwitch(deps.stateDir(), capabilities)) void deps.applyEngine().catch(() => undefined);
read = true;
+ fresh = capabilities;
} catch {
- // A failed read keeps the last answer.
+ // A failed read keeps the last answer (`read` false: never enough for the person's real screen).
}
- return jsonResponse(await state(read));
+ return jsonResponse(await state(read, fresh));
});
// Host and event: only Electron main (its per-launch secret, held in this process); the client token is
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256
index 7b41aa30..8164d14b 100644
--- a/apps/server/src/computer-use/shared/PARITY.sha256
+++ b/apps/server/src/computer-use/shared/PARITY.sha256
@@ -2,15 +2,15 @@
# A change to one must be ported to the other and this list updated in both.
da1cc6c26f3bf6758d3e2ce70e87bce383f68ee0ec6c85bbdc33b1f9a6c407fa contract.mjs
30cfcead85a5b2b54005ba6a0a67a43d9950de944b4615861ce9342bf043d4a4 contract.d.mts
-67776fea41ee567f1fe14b41ad9869f122ede65c36d32ccc3d23d8ec5c09c2c9 policy.mjs
-7dd420982f2731199827d5467851e84172d529f905e9d85c7873ceb26327e04a policy.d.mts
+4e8ade9f4ab4bc0efef8c30ac97f8fcd04eac16b5467e26880fe87c0b6e28c20 policy.mjs
+2dbfd8c20996603445f8dc6ff30c86dddd77426d4c123cc27d99361dc2f7c50a policy.d.mts
da515baa8aec2b533685d20c2be7ae007acc6b116db0386f8180e6779cb80f5c image.mjs
7c1a3224b7efd215656a94c38cf3152fe7a2444f25a6162988528978acf5458d image.d.mts
e47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjs
eabdf78c56733577d05aad490d9d7d5ece6c22e9b3ea9e6e9803d1a4a546fdd4 media.d.mts
a0bd964a5d0c7c5e26b4ce4ed8e8300f80a0a3a9da5d7f49c36f6ac2c907e934 native-worker.mjs
-d7414bbbb57a2d18ab72fce26e900444106051ce03f28d468b6139388f453bbb portable-host.mjs
-40cc047cb02f4c50ac98508bab9c0f4e1ee80c1d40b4b05a472a11ce01e8a153 portable-host.d.mts
+9f65946774cd07e40eccbaa23796d025203cac450bd6e7c7dc3a49369b72dbcc portable-host.mjs
+d2a8f2f2a997f1494e0f41015c94533a964b16b28bc5a7081780b2326aba818a portable-host.d.mts
7e9b4192b5e776b808be1f55e06a7b8ae53f1a575398d066641241ce9429a0c2 virtual-host.mjs
5bee4b675171a7cc8f02f172ba4b4d57171bb8ee27631974a5bfe33ed0bf58c8 virtual-host.d.mts
fc8bb13fd77f224d5a8a03d618e7253f877d4d678c21d245cd5fe91d45df097a virtual-worker.mjs
diff --git a/apps/server/src/computer-use/shared/policy.d.mts b/apps/server/src/computer-use/shared/policy.d.mts
index d53eca8a..c39b3718 100644
--- a/apps/server/src/computer-use/shared/policy.d.mts
+++ b/apps/server/src/computer-use/shared/policy.d.mts
@@ -14,6 +14,7 @@ export type ComputerUseBlock = {
os: string[] | null;
models: string[] | null;
real_screen: boolean;
+ real_screen_opt_in: boolean;
virtual_display: boolean;
deny_apps: DenyEntry[];
limits: Limits;
@@ -36,10 +37,13 @@ export const REAL_SCREEN_SETTING: string;
export const REAL_SCREEN_ENV: string;
export const REAL_SCREEN_OFF_MESSAGE: string;
export const REAL_SCREEN_FILE: string;
+export function serverRealScreen(block: unknown): boolean;
+export const REAL_SCREEN_NOT_OPTED_IN_MESSAGE: string;
+export const REAL_SCREEN_UNVERIFIED_MESSAGE: string;
export const REAL_SCREEN_DENY_RULES: Readonly<{ bash: Readonly<Record<string, "deny">>; edit: Readonly<Record<string, "deny">> }>;
export function realScreenOptIn(options?: { env?: Record<string, string | undefined>; saved?: unknown; allowEnvOn?: boolean }): boolean;
-export type SessionRules = { open: true; asks: false } | { open: false; asks: false; code: "virtual_unavailable" | "real_screen_unavailable" | "real_screen_off"; message: string };
-export function sessionRules(options?: { display?: "virtual" | "real"; server?: { real_screen?: boolean; virtual_display?: boolean }; optIn?: boolean }): SessionRules;
+export type SessionRules = { open: true; asks: false } | { open: false; asks: false; code: "virtual_unavailable" | "real_screen_unavailable" | "real_screen_unverified" | "real_screen_off"; message: string };
+export function sessionRules(options?: { display?: "virtual" | "real"; server?: { real_screen?: boolean; real_screen_opt_in?: boolean; virtual_display?: boolean }; optIn?: boolean; verified?: boolean }): SessionRules;
export const DENIED_APPS: Readonly<Record<"self" | "terminal" | "password_manager" | "credentials" | "lock_screen", readonly string[]>>;
export const SETTINGS_APPS: readonly string[];
diff --git a/apps/server/src/computer-use/shared/policy.mjs b/apps/server/src/computer-use/shared/policy.mjs
index bcc40e6c..00d60026 100644
--- a/apps/server/src/computer-use/shared/policy.mjs
+++ b/apps/server/src/computer-use/shared/policy.mjs
@@ -1,11 +1,16 @@
// Computer use, policy: the server switch (capabilities `computer_use`), the
-// person's one-time real-screen setting, the apps the agent may never open,
-// and (for the desktop's built-in browser only) which actions ask first.
+// person's real-screen opt-in (held by the server per account, set only from
+// the web console) and the device's own real-screen setting, the apps the
+// agent may never open, and (for the desktop's built-in browser only) which
+// actions ask first.
//
// Computer use itself never asks (owner decision 2026-10-21): the agent's own
// display needs no approval at all, and the person's real screen is a
// setting the person turns on once (off by default), then runs with no
-// per-session or per-action questions. What stays: the indicator and Stop,
+// per-session or per-action questions. The real screen needs BOTH the server
+// (`real_screen` and `real_screen_opt_in` true for this account, from a read
+// made just now) and this device's own setting: the server saying no, or not
+// answering, always wins; the device's setting can only keep it off. What stays: the indicator and Stop,
// the deny list and hit-testing, stuck-input release, the server switch.
//
// The same file ships in omnirush-cli (src/computer-use/) and omnirush-gui
@@ -40,12 +45,15 @@ function text(value, limit = 400) {
* The `computer_use` block of a /collect/capabilities answer, normalized. A
* missing or malformed block is off. The server already applied
* `rollout_percent` (and the account) to `enabled`.
- * Shape: { enabled, clients, os, models, real_screen, virtual_display,
- * deny_apps, limits, message, ask_send_delete }
- * (null lists: no restriction).
+ * Shape: { enabled, clients, os, models, real_screen, real_screen_opt_in,
+ * virtual_display, deny_apps, limits, message, ask_send_delete }
+ * (null lists: no restriction). `real_screen` (and `real_screen_opt_in`, the
+ * same value) is true only when the block says both `real_screen: true` and
+ * `real_screen_opt_in: true` (the account opted in from the web console):
+ * anything else, a missing key or an older server included, is off.
*/
export function normalizeComputerUse(block) {
- if (!isRecord(block)) return { enabled: false, clients: null, os: null, models: null, real_screen: false, virtual_display: false, deny_apps: [], limits: normalizeLimits(null), message: null, ask_send_delete: false };
+ if (!isRecord(block)) return { enabled: false, clients: null, os: null, models: null, real_screen: false, real_screen_opt_in: false, virtual_display: false, deny_apps: [], limits: normalizeLimits(null), message: null, ask_send_delete: false };
const denyApps = Array.isArray(block.deny_apps)
? block.deny_apps.flatMap((item) => {
if (typeof item === "string" && item.trim()) return [{ match: item.trim().slice(0, 200), reason: null }];
@@ -58,7 +66,8 @@ export function normalizeComputerUse(block) {
clients: strings(block.clients, 8, 40),
os: strings(block.os, 8, 40),
models: strings(block.models, 64, 128),
- real_screen: block.real_screen !== false,
+ real_screen: serverRealScreen(block),
+ real_screen_opt_in: serverRealScreen(block),
virtual_display: block.virtual_display !== false,
deny_apps: denyApps,
limits: normalizeLimits(block.limits),
@@ -81,7 +90,8 @@ export function computerUseSetting({ env = process.env, block, client, platform
const server = normalizeComputerUse(block);
const asked = String(env?.OMNIRUSH_COMPUTER_USE ?? "").trim().toLowerCase();
if (OFF.includes(asked)) return { ...server, enabled: false, source: "env", reason: "env" };
- if (ON.includes(asked)) return { ...server, enabled: true, real_screen: true, virtual_display: true, source: "env", reason: null };
+ // The environment turns computer use on (tests, support), never the person's real screen: that stays the server's.
+ if (ON.includes(asked)) return { ...server, enabled: true, virtual_display: true, source: "env", reason: null };
if (!isRecord(block)) return { ...server, enabled: false, source: "default", reason: "no_switch" };
if (!server.enabled) return { ...server, source: "server", reason: "off" };
if (server.clients && client && !server.clients.includes(client)) return { ...server, enabled: false, source: "server", reason: "client" };
@@ -103,6 +113,20 @@ export function offMessage(setting) {
// ------------------------------------------------------------- real screen
+/**
+ * Whether the server lets this account use the person's real screen: `real_screen` (the global switch) and
+ * `real_screen_opt_in` (the account's own opt-in, set only from the web console) both exactly true. Works on a
+ * raw capabilities block and on a normalized one. Anything else (false, missing, malformed, no block) is no.
+ */
+export function serverRealScreen(block) {
+ return isRecord(block) && block.real_screen === true && block.real_screen_opt_in === true;
+}
+
+/** Said to the agent when the account has not opted in (or the server turned it off). */
+export const REAL_SCREEN_NOT_OPTED_IN_MESSAGE = "Computer use on the person's own screen is not turned on for this account. Work on your own display instead (app_id \"virtual:main\"). Only the person can turn it on, in their account; never try to change it and never ask to.";
+/** Said to the agent when the server could not confirm it just now. */
+export const REAL_SCREEN_UNVERIFIED_MESSAGE = "The person's screen cannot be used right now: the account's setting could not be confirmed with the server. Work on your own display instead (app_id \"virtual:main\").";
+
/** The person's real screen is off until the person turns it on (Settings / `omnirush computer real-screen on`). */
export const REAL_SCREEN_DEFAULT = false;
/** The key both apps save the person's choice under, in <state or profile>/computer-use/real-screen.json. */
@@ -136,13 +160,16 @@ export function realScreenOptIn({ env = process.env, saved, allowEnvOn = false }
/**
* How a session opens, by display: `virtual` (the agent's own display) always opens with no question;
- * `real` (the person's screen) opens with no question when the server allows it and the person turned it
- * on, and is refused otherwise (never asked). Acts never ask on either.
+ * `real` (the person's screen) opens with no question only when the server allows it for this account
+ * (`serverRealScreen`), that answer was read from the server just now (`verified` true: a read that failed or
+ * was not made is no), and this device's own setting is on (`optIn`); it is refused otherwise (never asked).
+ * Acts never ask on either.
* Returns { open: true, asks: false } or { open: false, asks: false, code, message }.
*/
-export function sessionRules({ display, server = {}, optIn = REAL_SCREEN_DEFAULT } = {}) {
+export function sessionRules({ display, server = {}, optIn = REAL_SCREEN_DEFAULT, verified = false } = {}) {
if (display === "virtual") return server.virtual_display === false ? { open: false, asks: false, code: "virtual_unavailable", message: "The agent's own display is turned off for this account." } : { open: true, asks: false };
- if (server.real_screen === false) return { open: false, asks: false, code: "real_screen_unavailable", message: "Computer use on the person's screen is turned off for this account. Use your own display (app_id \"virtual:main\")." };
+ if (!serverRealScreen(server)) return { open: false, asks: false, code: "real_screen_unavailable", message: REAL_SCREEN_NOT_OPTED_IN_MESSAGE };
+ if (verified !== true) return { open: false, asks: false, code: "real_screen_unverified", message: REAL_SCREEN_UNVERIFIED_MESSAGE };
if (optIn !== true) return { open: false, asks: false, code: "real_screen_off", message: REAL_SCREEN_OFF_MESSAGE };
return { open: true, asks: false };
}
diff --git a/apps/server/src/computer-use/shared/portable-host.d.mts b/apps/server/src/computer-use/shared/portable-host.d.mts
index 070cf632..a1f6f68a 100644
--- a/apps/server/src/computer-use/shared/portable-host.d.mts
+++ b/apps/server/src/computer-use/shared/portable-host.d.mts
@@ -20,6 +20,8 @@ export type HostSettings = {
enabled: boolean;
/** The server allows the person's screen. */
real_screen?: boolean;
+ /** The account opted in from the web console (both this and real_screen true: the server allows it). */
+ real_screen_opt_in?: boolean;
/** The person turned the real screen on (a setting, off by default): sessions there open and act with no question. */
real_screen_on?: boolean;
/** The server allows the agent's own display. */
@@ -85,6 +87,8 @@ export type PortableHost = {
listen(dir: string): Promise<{ discoveryPath: string; externalDiscoveryPath: string; close(): Promise<void> }>;
control(input: { session_id: string; action: "stop" | "takeover" | "continue" | "hide" | "show" }): Promise<void>;
stopAll(by?: SessionEvent["by"], event?: SessionEvent["event"]): Promise<void>;
+ /** Ends the sessions on the person's screen only. */
+ stopRealScreen(by?: SessionEvent["by"], event?: SessionEvent["event"]): Promise<void>;
syntheticInput(withinMs?: number): Promise<boolean>;
/** A physical key or button press within `withinMs` (not injected, not a remote-debugging click). */
/** Default 1.5 s; a pointer press must be inside `rect` (screen pixels) when given; a key press must be one of `keys` (default Enter, Space). */
diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjs
index 143727e6..9fe9e3e7 100644
--- a/apps/server/src/computer-use/shared/portable-host.mjs
+++ b/apps/server/src/computer-use/shared/portable-host.mjs
@@ -38,7 +38,7 @@ import {
} from "./contract.mjs";
import { compareGrids, encodeJpeg, encodePng, fitLongSide, luminanceGrid, paintBoxes } from "./image.mjs";
import { sha256 } from "./media.mjs";
-import { deniedApp, sessionRules } from "./policy.mjs";
+import { deniedApp, REAL_SCREEN_NOT_OPTED_IN_MESSAGE, serverRealScreen, sessionRules } from "./policy.mjs";
const RPC = new Set(["initialize", "ping", "tools/list", "tools/call", "notifications/initialized", "notifications/cancelled"]);
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -129,9 +129,12 @@ function describeAction(a) {
* The host. Options:
* - native: a createNativeWorker() (or the same interface)
* - spool: a media spool (media.mjs createMediaSpool) frames go to, or null
- * - settings(): { enabled, real_screen, real_screen_on, virtual_display, limits, deny_apps } now
- * (real_screen: the server allows the person's screen; real_screen_on: the person turned it on)
- * - refresh(): read the server's switch and limits now (before each open); false or a throw: unread
+ * - settings(): { enabled, real_screen, real_screen_opt_in, real_screen_on, virtual_display, limits, deny_apps } now
+ * (real_screen and real_screen_opt_in: the server allows the person's screen for this account, both true;
+ * real_screen_on: this device's own setting is on)
+ * - refresh(): read the server's switch and limits now (before each open and each discover of the person's
+ * screen); true only when the server answered. Without it, or when it is false or throws, the person's
+ * screen is refused (the agent's own display is not affected)
* - context(): { ancestors, self_pids, tty_pids, chat } now, or a promise of it (chat: per-chat counts)
* - virtual: a createVirtualHost() for app_id "virtual:<name>", or null
* - onEvent(event): `computer.session` events; onState(states); onFrame(frame)
@@ -212,6 +215,8 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
}
const ticker = setInterval(() => {
const l = limits();
+ // The server no longer allows the person's screen for this account (its latest answer): those sessions end.
+ if (sessions.size && !serverRealScreen(settings() ?? {})) { void stopRealScreen("server"); return; }
for (const s of [...sessions.values()]) {
if (Date.now() >= s.expiresAt && s.phase !== "approval") void stop(s, "expired", "limit");
else if (s.phase === "working" && Date.now() - s.lastOperation > l.idle_pause_seconds * 1000) pause(s, "limit", "Paused after a while without work. Choose Continue when ready.");
@@ -401,13 +406,18 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
return mcpResult({ ok: true, os, contract: CONTRACT_VERSION, screens: screens.map((s) => ({ id: String(s.id), width: s.width, height: s.height, scale: s.scale, primary: Boolean(s.primary) })), apps: [...apps.values()], modes: ["observe", "control"], real_screen: true, virtual_display: virtualOk, ...(virtualOk ? { virtual_hint: virtual.hint } : {}) });
}
- async function openSession(connection, args) {
- // The server's switch and limits as of now, so a stricter value (off) applies at once.
- if (typeof refresh === "function") await Promise.resolve().then(refresh).catch(() => false);
+ /** The server's switch read just now: true only when it answered (no refresh, a false or a throw: not read). */
+ async function freshRead() {
+ if (typeof refresh !== "function") return false;
+ return Promise.resolve().then(refresh).then((ok) => ok === true, () => false);
+ }
+
+ async function openSession(connection, args, verified) {
+ // The server's switch and limits as of now (`call` read them), so a stricter value (off) applies at once.
const config = settings() ?? {};
const ctx = (await context()) ?? {};
if (!config.enabled) return mcpResult(failure("disabled", config.message || "Computer use is turned off."));
- const rules = sessionRules({ display: "real", server: config, optIn: config.real_screen_on === true });
+ const rules = sessionRules({ display: "real", server: config, optIn: config.real_screen_on === true, verified });
if (!rules.open) return mcpResult(failure(rules.code, rules.message));
if (lease) return mcpResult(failure("session_busy", "Another computer use session is open. Close it first."));
const windows = (await native.call("list")).filter((w) => w.appId === args.app_id && (args.pid === undefined || w.pid === args.pid));
@@ -462,14 +472,17 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
return virtual.call(name, args);
}
// The person's screen is off (the default): refused (and its windows not listed) before any desktop access.
+ // It needs the server's yes for this account read just now (a cached answer, or none, is not enough).
+ let verified = false;
+ if (name === "computer_open_session" || name === "computer_discover") verified = await freshRead();
if (name === "computer_open_session") {
const config = settings() ?? {};
- const rules = sessionRules({ display: "real", server: config, optIn: config.real_screen_on === true });
+ const rules = sessionRules({ display: "real", server: config, optIn: config.real_screen_on === true, verified });
if (!rules.open) return mcpResult(failure(rules.code, rules.message));
}
if (name === "computer_discover") {
const config = settings() ?? {};
- const rules = sessionRules({ display: "real", server: config, optIn: config.real_screen_on === true });
+ const rules = sessionRules({ display: "real", server: config, optIn: config.real_screen_on === true, verified });
if (rules.open && nativeDown) await ensureNative();
const why = !rules.open ? rules.message : !availability.supported || nativeDown ? availability.error ?? "Desktop access is unavailable." : null;
if (why) return mcpResult({ ok: true, os, contract: CONTRACT_VERSION, real_screen: false, real_screen_unavailable: why, screens: [], apps: [], modes: ["observe", "control"], virtual_display: Boolean(virtual) && config.virtual_display !== false, ...(virtual ? { virtual_hint: virtual.hint } : {}) });
@@ -477,9 +490,14 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
if (nativeDown && !(await ensureNative())) return mcpResult(failure("native_unavailable", "Desktop access stopped. Try again in a few minutes, or restart the app."));
if (!availability.supported) return mcpResult(failure(availability.code ?? "desktop_unavailable", availability.error ?? "Computer use is unavailable on this desktop."));
if (name === "computer_discover") return discover();
- if (name === "computer_open_session") return openSession(connection, args);
+ if (name === "computer_open_session") return openSession(connection, args, verified);
const s = sessions.get(args.session_id);
if (!s || s.connection !== connection) return mcpResult(failure("session_unavailable", "This session has ended. Open a new one."));
+ // The server's latest answer no longer allows the person's screen: the session ends before anything else.
+ if (!serverRealScreen(settings() ?? {}) && name !== "computer_close_session") {
+ await stop(s, "stopped", "server");
+ return mcpResult(failure("real_screen_unavailable", REAL_SCREEN_NOT_OPTED_IN_MESSAGE));
+ }
if (name === "computer_close_session") { await stop(s, "closed", "agent"); return mcpResult({ ok: true, session_id: s.id, state: "closed" }); }
if (name === "computer_session_status") {
const l = limits();
@@ -615,6 +633,11 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
await virtual?.stopAll(by, event);
}
+ /** Ends the sessions on the person's screen only (the agent's own display keeps running). */
+ async function stopRealScreen(by = "server", event = "stopped") {
+ for (const s of [...sessions.values()]) await stop(s, event, by);
+ }
+
/** Whether the latest press (key or button) within `withinMs` was synthetic (injected by a program). */
/**
* Whether a physical key or button press (not injected by a program, not a remote-debugging
@@ -652,5 +675,5 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
update();
const allState = () => [...publicState(), ...(virtual ? virtual.state() : [])];
- return { get availability() { return availability; }, os, connect, listen, control, stopAll, syntheticInput, physicalInput, watchInput, state: allState, close };
+ return { get availability() { return availability; }, os, connect, listen, control, stopAll, stopRealScreen, syntheticInput, physicalInput, watchInput, state: allState, close };
}
diff --git a/packages/types/src/desktop-ipc.ts b/packages/types/src/desktop-ipc.ts
index 801f04cf..9c3efc49 100644
--- a/packages/types/src/desktop-ipc.ts
+++ b/packages/types/src/desktop-ipc.ts
@@ -928,6 +928,7 @@ export type ComputerUseStatus = {
};
/** The person's one-time setting for their own screen (off by default). forced: the environment decides; allowed: the server allows it. */
+/** `on`: this computer's own setting; `allowed`: the server's yes for the account (its opt-in, set only in the web console), read just now. Both are needed. */
export type ComputerUseRealScreen = { on: boolean; forced: boolean; allowed: boolean };
// ---------------------------------------------------------------------------