Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

feat: desktop computer use built in

ogsamrat86bdd09e5714
diff --git a/apps/app/src/app/constants.ts b/apps/app/src/app/constants.ts
index f40dff59..42386a11 100644
--- a/apps/app/src/app/constants.ts
+++ b/apps/app/src/app/constants.ts
@@ -11,8 +11,8 @@ import {
type OmniRushExtensionPlatform,
} from "./extensions";
-/** Computer Use is paused for now: hidden from the Library, never reconnected, and the desktop refuses to start it. */
-export const COMPUTER_USE_PAUSED = true;
+/** Computer use is built into the app (server switch); the old Library connection stays off. */
+export const COMPUTER_USE_LIBRARY_CONNECT = false;
export const MODEL_PREF_KEY = "omnirush.defaultModel";
export const SESSION_MODEL_PREF_KEY = "omnirush.sessionModels";
@@ -236,8 +236,8 @@ export const MCP_QUICK_CONNECT: McpDirectoryInfo[] = [
// from the default catalog; "Show hidden" reveals it.
defaultHidden: true,
},
- // Computer Use is paused (COMPUTER_USE_PAUSED): not offered in the Library.
- ...BUILT_IN_OMNIRUSH_EXTENSION_MANIFESTS.filter((manifest) => !(COMPUTER_USE_PAUSED && manifest.id === "computer-use")).map(extensionManifestToDirectoryInfo),
+ // The Computer Use Library connection is off (COMPUTER_USE_LIBRARY_CONNECT): not offered in the Library.
+ ...BUILT_IN_OMNIRUSH_EXTENSION_MANIFESTS.filter((manifest) => !(!COMPUTER_USE_LIBRARY_CONNECT && manifest.id === "computer-use")).map(extensionManifestToDirectoryInfo),
];
export const OMNIRUSH_EXTENSION_CATALOG = MCP_QUICK_CONNECT.filter((entry) => entry.kind === "extension");
diff --git a/apps/app/src/app/lib/desktop.ts b/apps/app/src/app/lib/desktop.ts
index 2d7efdd9..c433fc12 100644
--- a/apps/app/src/app/lib/desktop.ts
+++ b/apps/app/src/app/lib/desktop.ts
@@ -56,6 +56,7 @@ import type {
OpenBrowserUrlResult,
} from "@omnirush/browser-tabs";
import type { ImportableSite, ImportSourceAvailability } from "@omnirush/browser-logins";
+import type { ComputerUseStatus } from "@omnirush/types/desktop-ipc";
export type BrowserLoginSite = ImportableSite;
@@ -189,6 +190,9 @@ declare global {
automationRunner?: {
onCredentialRejected?: (callback: () => void) => () => void;
};
+ computerUse?: {
+ onStatus?: (callback: (status: ComputerUseStatus) => void) => () => void;
+ };
fileSystem?: {
getPathForFile?: (file: File) => string;
};
diff --git a/apps/app/src/components/chat/message-list.tsx b/apps/app/src/components/chat/message-list.tsx
index 4a9d6859..676e1d94 100644
--- a/apps/app/src/components/chat/message-list.tsx
+++ b/apps/app/src/components/chat/message-list.tsx
@@ -35,6 +35,7 @@ import { openTargetFromUrl } from "@/react-app/domains/session/artifacts/open-ta
import { sessionErrorPresentationFromUIMessage } from "@/react-app/domains/session/sync/session-error"
import { ApplyPatchTool } from "@/components/tools/apply-patch"
import { BashTool } from "@/components/tools/bash"
+import { ComputerUseToolLine } from "@/components/tools/computer-use"
import { EditTool } from "@/components/tools/edit"
import { EnvVarRequestTool } from "@/components/tools/env-var-request"
import { ReadFileTool, WriteFileTool } from "@/components/tools/file"
@@ -112,6 +113,7 @@ import {
taskChildSessionId,
isTodoWriteToolPart,
isWebFetchToolPart,
+ isComputerUseToolPart,
isWebSearchToolPart,
isHostedWebSearchToolPart,
isWriteToolPart,
@@ -161,6 +163,8 @@ function MessageTimestamp({ message, className }: { message: UIMessage; classNam
interface ToolMessageProps {
part: ToolUIPart | DynamicToolUIPart
+ /** Screenshots that came with the call (computer use). */
+ attachments?: FileUIPart[]
}
/**
@@ -187,11 +191,11 @@ class ToolMessage extends React.Component<ToolMessageProps, { failed: boolean }>
<div className="text-xs text-muted-foreground">Tool step unavailable</div>
)
}
- return <ToolMessageInner part={this.props.part} />
+ return <ToolMessageInner part={this.props.part} attachments={this.props.attachments} />
}
}
-const ToolMessageInner = ({ part }: ToolMessageProps) => {
+const ToolMessageInner = ({ part, attachments }: ToolMessageProps) => {
const { connectorIdentities, onMcpReconnect, onMcpReopenAuthorization, onMcpRetry } = useMessageList()
const parentRun = React.useContext(ParentRunContext)
const resolveLifecycle = useCurrentToolLifecycleResolver()
@@ -317,6 +321,10 @@ const ToolMessageInner = ({ part }: ToolMessageProps) => {
return <OmniRushAutomationProposalTool part={part} />
}
+ if (isComputerUseToolPart(part)) {
+ return <ComputerUseToolLine part={part} attachments={attachments} />
+ }
+
// Failed calls use the same sentence line with the "failures are
// instructions" treatment (inline Reconnect/Retry).
if (part.type === "dynamic-tool") {
@@ -575,7 +583,7 @@ const AssistantMessage = React.memo(
return (
<div key={`tool-${index}`} className="w-full">
- <ToolMessage part={group.part} />
+ <ToolMessage part={group.part} attachments={group.attachments} />
</div>
)
})}
diff --git a/apps/app/src/components/chat/utils.ts b/apps/app/src/components/chat/utils.ts
index 8fdf59c1..f8f54c53 100644
--- a/apps/app/src/components/chat/utils.ts
+++ b/apps/app/src/components/chat/utils.ts
@@ -3,6 +3,7 @@ import type { ThreadStatus } from "@/lib/messages"
// Relative so the pure grouping contract stays importable from alias-free
// test workspaces (evals specs import this module directly).
import { isAggregatableToolPart, type AggregateThought } from "../../lib/tool-aggregate"
+import { isComputerUseToolPart } from "../../lib/build-in-tools"
const DOCX_MIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
const PPTX_MIME = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
@@ -171,7 +172,7 @@ type AssistantRenderGroup =
| { kind: "text"; text: string }
| { kind: "reasoning"; text: string; isStreaming: boolean }
| { kind: "file"; part: FileUIPart }
- | { kind: "tool"; part: ToolUIPart | DynamicToolUIPart }
+ | { kind: "tool"; part: ToolUIPart | DynamicToolUIPart; attachments?: FileUIPart[] }
| { kind: "tool-aggregate"; parts: (ToolUIPart | DynamicToolUIPart)[]; thoughts: AggregateThought[] }
/**
@@ -310,6 +311,12 @@ export function getAssistantRenderGroups(
}
if (part.type === "file") {
+ // A computer use screenshot shows as the tool line's thumbnail.
+ const previous = groups.at(-1)
+ if (previous?.kind === "tool" && isComputerUseToolPart(previous.part) && part.mediaType.startsWith("image/")) {
+ previous.attachments = [...(previous.attachments ?? []), part]
+ continue
+ }
groups.push({ kind: "file", part })
continue
}
diff --git a/apps/app/src/components/tools/computer-use.tsx b/apps/app/src/components/tools/computer-use.tsx
new file mode 100644
index 00000000..2265911a
--- /dev/null
+++ b/apps/app/src/components/tools/computer-use.tsx
@@ -0,0 +1,124 @@
+import { AppWindow, Camera, Hand, LoaderCircle, LogOut, MonitorSmartphone, Search } from "lucide-react"
+import type { FileUIPart } from "ai"
+
+import { Image } from "@/components/ui/image"
+import type { ComputerUseToolPart } from "@/lib/build-in-tools"
+import { describeComputerUseAction } from "@/react-app/domains/session/surface/computer-use/describe"
+
+/** Results that are part of the normal flow, not failures. */
+const NEUTRAL_CODES = new Set(["stale_observation", "paused_by_person", "approval_denied"])
+
+const NEUTRAL_LABELS: Record<string, string> = {
+ stale_observation: "Looked again",
+ paused_by_person: "Paused",
+ approval_denied: "Not allowed",
+}
+
+type ComputerUseResult = { ok: boolean; code: string | null; message: string | null; appName: string | null }
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value)
+}
+
+function stringValue(value: unknown): string | null {
+ return typeof value === "string" && value.trim() ? value.trim() : null
+}
+
+/** The JSON a computer use tool answers with ({ok, code, message, ...}), or null. */
+export function parseComputerUseResult(output: unknown): ComputerUseResult | null {
+ let value = output
+ if (typeof output === "string") {
+ try {
+ value = JSON.parse(output)
+ } catch {
+ return null
+ }
+ }
+ if (!isRecord(value) || typeof value.ok !== "boolean") return null
+ const window = isRecord(value.window) ? value.window : null
+ return {
+ ok: value.ok,
+ code: stringValue(value.code),
+ message: stringValue(value.message),
+ appName: stringValue(window?.app_name),
+ }
+}
+
+/** The short label for one computer use call. */
+export function computerUseToolLabel(tool: string, input: unknown, appName: string | null): string {
+ switch (tool) {
+ case "computer_discover":
+ return "Looked at open apps"
+ case "computer_open_session":
+ return `Asked to use ${appName ?? "an app"}`
+ case "computer_observe":
+ return "Took a screenshot"
+ case "computer_act":
+ return describeComputerUseAction(isRecord(input) ? input.action : null)
+ case "computer_session_status":
+ return "Checked the session"
+ case "computer_close_session":
+ return "Ended computer use"
+ default:
+ return "Used the computer"
+ }
+}
+
+function ToolIcon(props: { tool: string }) {
+ const className = "size-4 shrink-0"
+ switch (props.tool) {
+ case "computer_discover":
+ return <Search className={className} />
+ case "computer_open_session":
+ return <AppWindow className={className} />
+ case "computer_observe":
+ return <Camera className={className} />
+ case "computer_act":
+ return <Hand className={className} />
+ case "computer_close_session":
+ return <LogOut className={className} />
+ default:
+ return <MonitorSmartphone className={className} />
+ }
+}
+
+/** One computer use step in the transcript: what it did, why, how it went, and its screenshot. */
+export function ComputerUseToolLine(props: { part: ComputerUseToolPart; attachments?: FileUIPart[] }) {
+ const { part } = props
+ const tool = part.toolName.slice("computer-use_".length)
+ const input = isRecord(part.input) ? part.input : null
+ const intent = stringValue(input?.intent)
+ const result = part.state === "output-available" ? parseComputerUseResult(part.output) : null
+ const running = part.state === "input-streaming" || part.state === "input-available"
+ const errorText = part.state === "output-error" ? part.errorText : null
+ // Screenshots arrive as data: URLs; stored media references are not loadable here.
+ const thumbnail = props.attachments?.find((file) => file.url.startsWith("data:image/"))
+ const label = computerUseToolLabel(tool, input, result?.appName ?? null)
+
+ return (
+ <div className="flex flex-col gap-1.5 text-sm" data-computer-use-tool={tool}>
+ <div className="flex min-w-0 items-center gap-2 text-muted-foreground">
+ {running ? <LoaderCircle className="size-4 shrink-0 animate-spin" /> : <ToolIcon tool={tool} />}
+ <span className="min-w-0 truncate text-foreground">{label}</span>
+ {result && !result.ok && result.code ? (
+ NEUTRAL_CODES.has(result.code) ? (
+ <span className="shrink-0 rounded-full border border-border px-2 text-[11px] text-muted-foreground" title={result.message ?? undefined}>
+ {NEUTRAL_LABELS[result.code]}
+ </span>
+ ) : (
+ <span className="shrink-0 rounded-full bg-red-3 px-2 font-mono text-[11px] text-red-11" title={result.message ?? undefined}>
+ {result.code}
+ </span>
+ )
+ ) : null}
+ {errorText ? <span className="shrink-0 rounded-full bg-red-3 px-2 text-[11px] text-red-11" title={errorText}>Failed</span> : null}
+ </div>
+ {intent ? <div className="pl-6 text-xs text-muted-foreground">{intent}</div> : null}
+ {thumbnail ? (
+ <div className="pl-6">
+ <Image src={thumbnail.url} alt="Screenshot" previewMaxWidth={220} previewMaxHeight={124} className="rounded-lg border border-border/70" />
+ </div>
+ ) : null}
+ </div>
+ )
+}
diff --git a/apps/app/src/lib/build-in-tools.ts b/apps/app/src/lib/build-in-tools.ts
index 9ee472ed..823e01af 100644
--- a/apps/app/src/lib/build-in-tools.ts
+++ b/apps/app/src/lib/build-in-tools.ts
@@ -380,6 +380,13 @@ export function isHostedWebSearchToolPart(part: ToolUIPart | DynamicToolUIPart):
return part.type === "dynamic-tool" && part.toolName === "web_search";
}
+/** A tool of the app's built-in computer use server ("computer-use_computer_act", ...). */
+export type ComputerUseToolPart = DynamicToolUIPart & { toolName: `computer-use_${string}` };
+
+export function isComputerUseToolPart(part: ToolUIPart | DynamicToolUIPart): part is ComputerUseToolPart {
+ return part.type === "dynamic-tool" && part.toolName.startsWith("computer-use_");
+}
+
export type QuestionToolPart = BuiltInDynamicToolPart<"question", QuestionInput>;
export function isQuestionToolPart(part: ToolUIPart | DynamicToolUIPart): part is QuestionToolPart {
diff --git a/apps/app/src/react-app/domains/connections/store.ts b/apps/app/src/react-app/domains/connections/store.ts
index 218037a2..42eb4a2b 100644
--- a/apps/app/src/react-app/domains/connections/store.ts
+++ b/apps/app/src/react-app/domains/connections/store.ts
@@ -8,7 +8,7 @@ import { t } from "../../../i18n";
import {
getMcpServerName,
isBuiltInOmniRushExtension,
- COMPUTER_USE_PAUSED,
+ COMPUTER_USE_LIBRARY_CONNECT,
MCP_QUICK_CONNECT,
type McpDirectoryInfo,
} from "../../../app/constants";
@@ -230,7 +230,7 @@ export function createConnectionsStore(options: {
entries: McpServerEntry[],
observedStatuses: Record<string, unknown>,
) => {
- if (COMPUTER_USE_PAUSED || !isDesktopRuntime() || options.workspaceType() !== "local" || !directory) return;
+ if (!COMPUTER_USE_LIBRARY_CONNECT || !isDesktopRuntime() || options.workspaceType() !== "local" || !directory) return;
const entry = entries.find((candidate) => candidate.name === "computer-use" || candidate.id === "computer-use");
if (!entry || entry.config.type !== "local" || entry.config.enabled === false) return;
if (state.mcpConnectingName) return;
diff --git a/apps/app/src/react-app/domains/session/surface/computer-use-controls.tsx b/apps/app/src/react-app/domains/session/surface/computer-use-controls.tsx
index 5fcf3c08..4b6e3be4 100644
--- a/apps/app/src/react-app/domains/session/surface/computer-use-controls.tsx
+++ b/apps/app/src/react-app/domains/session/surface/computer-use-controls.tsx
@@ -1,63 +1,24 @@
-import { useState } from "react";
-import { useMutation, useQuery } from "@tanstack/react-query";
-import { z } from "zod";
-import { Button } from "@/components/ui/button";
+import { ComputerUseInline, isLiveComputerUseSession } from "./computer-use/inline";
+import { useComputerUseStatus, useInlineComputerUseChats } from "./computer-use/status-store";
-const sessionsSchema = z.array(z.object({
- connectionId: z.string(), id: z.string(), phase: z.enum(["approval", "working", "paused"]),
- appName: z.string(), task: z.string(), mode: z.enum(["observe", "assist", "control"]),
- recoverable: z.boolean().optional(),
- status: z.string().optional(), windowTitle: z.string().optional(), canContinue: z.boolean().optional(),
- previewVisible: z.boolean().optional(), remainingSeconds: z.number().optional(),
- windows: z.array(z.object({ id: z.number(), title: z.string() })).optional(),
-}));
+/**
+ * Fallback for computer use approvals and running sessions no open chat shows
+ * inline (another chat, or no chat on screen), so an approval and Stop are
+ * never out of sight.
+ */
export function ComputerUseControls() {
- const [windows, setWindows] = useState<Record<string, number>>({});
- const state = useQuery({ queryKey: ["computer-use", "host"], enabled: Boolean(window.__OMNIRUSH_ELECTRON__?.invokeDesktop),
- queryFn: async () => {
- const invoke = window.__OMNIRUSH_ELECTRON__?.invokeDesktop;
- return invoke ? sessionsSchema.parse(await invoke("getComputerUseState")) : [];
- }, refetchInterval: 750, refetchIntervalInBackground: true, retry: false });
- const action = useMutation({ mutationFn: async (value: { connectionId: string; id: string; action: string; windowId?: number }) => {
- const invoke = window.__OMNIRUSH_ELECTRON__?.invokeDesktop;
- if (!invoke) throw new Error("Computer Use requires the desktop app.");
- await invoke("computerUseAction", value);
- }, onSuccess: () => state.refetch() });
- if (!state.data?.length) return null;
- return <aside aria-label="Computer Use controls" className="fixed bottom-6 right-6 z-50 flex w-96 max-w-[calc(100vw-3rem)] flex-col gap-3">
- {state.data.map((session) => {
- const approval = session.phase === "approval";
- const blocked = session.phase === "paused" && session.recoverable !== true;
- const selectedWindow = windows[session.id] ?? session.windows?.[0]?.id;
- const send = (command: string) => action.mutate({ connectionId: session.connectionId, id: session.id, action: command, windowId: selectedWindow });
- if (!approval && !blocked) {
- return session.previewVisible ? null : <div key={session.connectionId} className="flex justify-end gap-2">
- <Button variant="outline" onClick={() => send("show")}>Show {session.appName} preview</Button>
- <Button variant="destructive" disabled={action.isPending} onClick={() => send("stop")}>Stop</Button>
- </div>;
- }
- return <section key={session.connectionId} className="rounded-2xl border border-border bg-background p-4 shadow-lg">
- <p className="text-sm font-semibold">{approval ? `Use ${session.appName}?` : `Computer Use · ${session.appName}`}</p>
- <p className="mt-1 text-sm text-muted-foreground">{session.task}</p>
- {approval ? <>
- <label className="mt-3 block text-xs font-medium">Window
- <select aria-label="Window to allow" className="mt-1 w-full rounded-md border border-border bg-background p-2 text-sm" value={selectedWindow} onChange={(event) => setWindows({ ...windows, [session.id]: Number(event.target.value) })}>
- {session.windows?.map((window) => <option key={window.id} value={window.id}>{window.title}</option>)}
- </select>
- </label>
- <p className="my-3 text-xs text-muted-foreground">{session.mode === "observe" ? "Read this window. No clicks or typing." : session.mode === "assist" ? "Read and use this window’s app controls." : "Use this window’s mouse and keyboard. Your input interrupts control. Stop ends access."} Access lasts up to 15 minutes. Window content goes to your selected model provider.</p>
- <div className="flex justify-end gap-2"><Button variant="outline" onClick={() => send("deny")}>Cancel</Button><Button disabled={selectedWindow === undefined || action.isPending} onClick={() => send("approve")}>Allow and start</Button></div>
- </> : <>
- <p className="mt-2 truncate text-xs text-muted-foreground">{session.windowTitle} · {session.mode === "observe" ? "Read only" : session.mode === "assist" ? "App controls" : "Mouse and keyboard"}</p>
- <p role="status" className="my-3 text-sm">{session.status}</p>
- <div className="flex flex-wrap gap-2">
- <Button variant="outline" disabled={action.isPending || session.canContinue === false} onClick={() => send("resume")}>Continue</Button>
- <Button variant="destructive" onClick={() => send("stop")}>Stop</Button>
-
- </div>
- </>}
- {action.error ? <p role="alert" className="mt-2 text-sm text-destructive">{action.error.message}</p> : null}
- </section>;
- })}
- </aside>;
+ const status = useComputerUseStatus();
+ const chats = useInlineComputerUseChats((state) => state.chats);
+ const unseen = (chat: string | null) => (chat === null ? chats.length === 0 : !chats.includes(chat));
+ const requests = status?.requests.filter((request) => unseen(request.chat)) ?? [];
+ const sessions = status?.sessions.filter((session) => isLiveComputerUseSession(session) && unseen(session.chat)) ?? [];
+ if (!requests.length && !sessions.length) return null;
+ return (
+ <aside
+ aria-label="Computer use"
+ className="fixed bottom-6 right-6 z-50 flex max-h-[calc(100vh-3rem)] w-[26rem] max-w-[calc(100vw-3rem)] flex-col overflow-y-auto rounded-2xl border border-dls-border bg-background shadow-lg"
+ >
+ <ComputerUseInline requests={requests} sessions={sessions} />
+ </aside>
+ );
}
diff --git a/apps/app/src/react-app/domains/session/surface/computer-use/describe.ts b/apps/app/src/react-app/domains/session/surface/computer-use/describe.ts
new file mode 100644
index 00000000..57087e6a
--- /dev/null
+++ b/apps/app/src/react-app/domains/session/surface/computer-use/describe.ts
@@ -0,0 +1,120 @@
+// Plain-words helpers for computer use: what an action does, where its
+// points land on a displayed frame, and which chat a request belongs to.
+
+export type ComputerUsePoint = { x: number; y: number };
+
+const MODIFIER_LABELS: Record<string, string> = { ctrl: "Ctrl", shift: "Shift", alt: "Alt", meta: "Meta" };
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function numberValue(value: unknown): number | null {
+ return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function modifiersOf(action: Record<string, unknown>): string[] {
+ const value = action.modifiers;
+ if (!Array.isArray(value)) return [];
+ return value.flatMap((item) => (typeof item === "string" ? [MODIFIER_LABELS[item.toLowerCase()] ?? item] : []));
+}
+
+function keyLabel(key: string): string {
+ return key.length === 1 ? key.toUpperCase() : key;
+}
+
+/** "Ctrl+V", "Enter", "Shift+Tab". */
+export function describeKeyChord(action: Record<string, unknown>): string {
+ const key = typeof action.key === "string" ? keyLabel(action.key) : "";
+ return [...modifiersOf(action), key].filter(Boolean).join("+");
+}
+
+/** The text a `type` action types, shown in full; null for other actions. */
+export function typedText(action: unknown): string | null {
+ return isRecord(action) && action.type === "type" && typeof action.text === "string" ? action.text : null;
+}
+
+/**
+ * One action in plain words: "Click at 120, 48", "Drag from 10, 10 to 200, 40",
+ * "Scroll down 3 steps at 400, 300", "Type \"hello\"", "Press Ctrl+V", "Wait 500 ms".
+ */
+export function describeComputerUseAction(action: unknown): string {
+ if (!isRecord(action) || typeof action.type !== "string") return "Do an action";
+ const x = numberValue(action.x);
+ const y = numberValue(action.y);
+ const at = x !== null && y !== null ? ` at ${Math.round(x)}, ${Math.round(y)}` : "";
+ const held = modifiersOf(action);
+ const withKeys = held.length ? ` holding ${held.join("+")}` : "";
+ switch (action.type) {
+ case "click":
+ return `Click${at}${withKeys}`;
+ case "double_click":
+ return `Double click${at}${withKeys}`;
+ case "right_click":
+ return `Right click${at}${withKeys}`;
+ case "move":
+ return `Move the pointer${at}`;
+ case "drag": {
+ const path = pointsOf(action);
+ const first = path[0];
+ const last = path.at(-1);
+ if (!first || !last) return `Drag${withKeys}`;
+ return `Drag from ${Math.round(first.x)}, ${Math.round(first.y)} to ${Math.round(last.x)}, ${Math.round(last.y)}${withKeys}`;
+ }
+ case "scroll": {
+ const delta = numberValue(action.delta) ?? 0;
+ const horizontal = action.axis === "horizontal";
+ const direction = horizontal ? (delta < 0 ? "left" : "right") : delta < 0 ? "up" : "down";
+ const steps = Math.abs(delta);
+ return `Scroll ${direction} ${steps} ${steps === 1 ? "step" : "steps"}${at}`;
+ }
+ case "type":
+ return `Type "${typeof action.text === "string" ? action.text : ""}"`;
+ case "key":
+ return `Press ${describeKeyChord(action)}`;
+ case "wait": {
+ const ms = numberValue(action.ms);
+ return ms === null ? "Wait" : `Wait ${Math.round(ms)} ms`;
+ }
+ default:
+ return "Do an action";
+ }
+}
+
+/** The points of an action (a drag's path, or its x, y), in frame pixels. */
+export function pointsOf(action: Record<string, unknown>): ComputerUsePoint[] {
+ if (Array.isArray(action.path)) {
+ return action.path.flatMap((point) => {
+ if (!isRecord(point)) return [];
+ const x = numberValue(point.x);
+ const y = numberValue(point.y);
+ return x === null || y === null ? [] : [{ x, y }];
+ });
+ }
+ const x = numberValue(action.x);
+ const y = numberValue(action.y);
+ return x === null || y === null ? [] : [{ x, y }];
+}
+
+/** Maps points from frame pixels to the displayed size of the frame image. */
+export function scalePoints(
+ points: ComputerUsePoint[],
+ frame: { width: number; height: number },
+ shown: { width: number; height: number },
+): ComputerUsePoint[] {
+ if (frame.width <= 0 || frame.height <= 0) return [];
+ const sx = shown.width / frame.width;
+ const sy = shown.height / frame.height;
+ return points.map((point) => ({ x: point.x * sx, y: point.y * sy }));
+}
+
+/** A request or session shows in a chat when it has no chat or that chat. */
+export function belongsToChat(chat: string | null, sessionId: string): boolean {
+ return chat === null || chat === sessionId;
+}
+
+/** "3 min", "less than a minute". */
+export function formatMinutesLeft(seconds: number): string {
+ if (seconds < 60) return "less than a minute";
+ return `${Math.floor(seconds / 60)} min`;
+}
diff --git a/apps/app/src/react-app/domains/session/surface/computer-use/inline.tsx b/apps/app/src/react-app/domains/session/surface/computer-use/inline.tsx
new file mode 100644
index 00000000..b3001f8c
--- /dev/null
+++ b/apps/app/src/react-app/domains/session/surface/computer-use/inline.tsx
@@ -0,0 +1,35 @@
+import type { ComputerUseRequest, ComputerUseSessionState } from "@omnirush/types/desktop-ipc";
+
+import { belongsToChat } from "./describe";
+import { ComputerUseLivePanel } from "./live-panel";
+import { ComputerUseRequestCard } from "./request-card";
+import { useComputerUseStatus, useRegisterInlineComputerUse } from "./status-store";
+
+export type ComputerUseForChat = {
+ requests: ComputerUseRequest[];
+ sessions: ComputerUseSessionState[];
+};
+
+export function isLiveComputerUseSession(session: ComputerUseSessionState): boolean {
+ return session.phase === "working" || session.phase === "paused";
+}
+
+/** The requests and running sessions this chat shows inline (and claims from the global fallback). */
+export function useComputerUseForChat(sessionId: string): ComputerUseForChat {
+ useRegisterInlineComputerUse(sessionId);
+ const status = useComputerUseStatus();
+ return {
+ requests: status?.requests.filter((request) => belongsToChat(request.chat, sessionId)) ?? [],
+ sessions: status?.sessions.filter((session) => isLiveComputerUseSession(session) && belongsToChat(session.chat, sessionId)) ?? [],
+ };
+}
+
+/** Running sessions first, then the approvals waiting for the person. */
+export function ComputerUseInline(props: ComputerUseForChat) {
+ return (
+ <>
+ {props.sessions.map((session) => <ComputerUseLivePanel key={session.id} session={session} />)}
+ {props.requests.map((request) => <ComputerUseRequestCard key={request.request_id} request={request} />)}
+ </>
+ );
+}
diff --git a/apps/app/src/react-app/domains/session/surface/computer-use/live-panel.tsx b/apps/app/src/react-app/domains/session/surface/computer-use/live-panel.tsx
new file mode 100644
index 00000000..a2635992
--- /dev/null
+++ b/apps/app/src/react-app/domains/session/surface/computer-use/live-panel.tsx
@@ -0,0 +1,100 @@
+import { useState } from "react";
+import { ChevronDown, ChevronRight, Hand, Play, Square } from "lucide-react";
+
+import type { ComputerUseControlAction, ComputerUseSessionState } from "@omnirush/types/desktop-ipc";
+import { Button } from "@/components/ui/button";
+import { stripRemoteMethodErrorPrefix } from "@/react-app/domains/settings/state/electron-updater-state";
+
+import { formatMinutesLeft } from "./describe";
+import { ComputerUseAppIdentity, ComputerUseFrame } from "./request-card";
+import { controlComputerUse } from "./status-store";
+
+/** A running (or paused) computer use session: what it sees, what it did, and Stop. */
+export function ComputerUseLivePanel(props: { session: ComputerUseSessionState }) {
+ const session = props.session;
+ const [open, setOpen] = useState(true);
+ const [busy, setBusy] = useState(false);
+ const [error, setError] = useState<string | null>(null);
+ const paused = session.phase === "paused";
+
+ const control = (action: ComputerUseControlAction) => {
+ setBusy(true);
+ setError(null);
+ controlComputerUse(session.id, action)
+ .catch((reason: unknown) => setError(reason instanceof Error ? stripRemoteMethodErrorPrefix(reason.message) : "That did not work. Try again."))
+ .finally(() => setBusy(false));
+ };
+
+ const buttons = (
+ <div className="flex shrink-0 flex-wrap items-center gap-2">
+ {paused && session.can_continue ? (
+ <Button type="button" variant="outline" size="xs" disabled={busy} onClick={() => control("continue")} data-computer-use-control="continue">
+ <Play data-icon="inline-start" />
+ Continue
+ </Button>
+ ) : null}
+ {!paused && session.mode === "control" ? (
+ <Button type="button" variant="outline" size="xs" disabled={busy} onClick={() => control("takeover")} data-computer-use-control="takeover">
+ <Hand data-icon="inline-start" />
+ Take over
+ </Button>
+ ) : null}
+ <Button type="button" variant="destructive" size="xs" disabled={busy} onClick={() => control("stop")} data-computer-use-control="stop">
+ <Square data-icon="inline-start" />
+ Stop
+ </Button>
+ </div>
+ );
+
+ return (
+ <div data-computer-use-session={session.id} className="flex flex-col gap-2 border-b border-dls-border px-4 py-2.5">
+ <div className="flex min-w-0 items-center gap-2">
+ <button
+ type="button"
+ className="flex size-5 shrink-0 items-center justify-center rounded text-dls-secondary hover:text-dls-text"
+ onClick={() => setOpen(!open)}
+ aria-expanded={open}
+ aria-label={open ? "Collapse computer use" : "Expand computer use"}
+ >
+ {open ? <ChevronDown size={14} /> : <ChevronRight size={14} />}
+ </button>
+ <ComputerUseAppIdentity app={session.app} icon={null} compact />
+ <span className="min-w-0 flex-1 truncate text-[12px] text-dls-secondary" title={session.window.title}>{session.window.title}</span>
+ <span
+ className={`shrink-0 rounded-full px-2 py-0.5 text-[11px] font-medium ${paused ? "bg-amber-3 text-amber-11" : "bg-green-3 text-green-11"}`}
+ >
+ {paused ? "Paused" : "Working"}
+ </span>
+ {open ? null : buttons}
+ </div>
+ {open ? (
+ <>
+ <div className="flex flex-wrap gap-x-3 text-[11px] leading-4 text-dls-secondary">
+ <span>{session.mode === "observe" ? "Sees the window" : "Uses mouse and keyboard"}</span>
+ <span>{session.actions_left} {session.actions_left === 1 ? "action" : "actions"} left</span>
+ <span>{formatMinutesLeft(session.seconds_left)} left</span>
+ </div>
+ {session.status ? <p role="status" className="text-[12px] leading-5 text-dls-text">{session.status}</p> : null}
+ {session.frame ? (
+ <ComputerUseFrame frame={session.frame} alt={`${session.app.app_name} window`} className="max-h-[220px]" />
+ ) : null}
+ {session.log.length ? (
+ <ol className="flex flex-col gap-0.5 text-[12px] leading-5">
+ {session.log.slice(-8).map((entry, index) => (
+ <li key={`${entry.at}-${index}`} className="flex min-w-0 items-baseline gap-2">
+ {entry.ok ? null : (
+ <span className="shrink-0 rounded bg-red-3 px-1 font-mono text-[10px] text-red-11">{entry.code ?? "failed"}</span>
+ )}
+ <span className="shrink-0 text-dls-text">{entry.action}</span>
+ {entry.intent ? <span className="truncate text-dls-secondary">{entry.intent}</span> : null}
+ </li>
+ ))}
+ </ol>
+ ) : null}
+ {buttons}
+ </>
+ ) : null}
+ {error ? <p role="alert" className="text-[12px] font-medium leading-5 text-red-11">{error}</p> : null}
+ </div>
+ );
+}
diff --git a/apps/app/src/react-app/domains/session/surface/computer-use/request-card.tsx b/apps/app/src/react-app/domains/session/surface/computer-use/request-card.tsx
new file mode 100644
index 00000000..02746631
--- /dev/null
+++ b/apps/app/src/react-app/domains/session/surface/computer-use/request-card.tsx
@@ -0,0 +1,199 @@
+import { useState } from "react";
+import { AppWindow, Check, Clock3, MonitorSmartphone, XCircle } from "lucide-react";
+
+import type { ComputerUseAnswer, ComputerUseAppRef, ComputerUseRequest } from "@omnirush/types/desktop-ipc";
+import { Button } from "@/components/ui/button";
+import { stripRemoteMethodErrorPrefix } from "@/react-app/domains/settings/state/electron-updater-state";
+
+import { describeComputerUseAction, scalePoints, typedText, type ComputerUsePoint } from "./describe";
+import { answerComputerUse } from "./status-store";
+
+const APPROVE_WITH_OWN_INPUT = "Use your own mouse or keyboard to approve.";
+
+/** The app the host verified: icon, name, executable or bundle id, process id. */
+export function ComputerUseAppIdentity(props: { app: ComputerUseAppRef; icon: string | null; compact?: boolean }) {
+ const icon = props.icon?.startsWith("data:image/") ? props.icon : null;
+ return (
+ <div className="flex min-w-0 items-center gap-2.5">
+ {icon ? (
+ <img src={icon} alt="" className={props.compact ? "size-5 shrink-0" : "size-8 shrink-0"} />
+ ) : (
+ <div className={`flex shrink-0 items-center justify-center rounded-lg border border-dls-border bg-dls-hover text-dls-secondary ${props.compact ? "size-5" : "size-8"}`}>
+ <AppWindow size={props.compact ? 12 : 16} />
+ </div>
+ )}
+ <div className="min-w-0">
+ <div className="truncate text-[13px] font-semibold leading-5 text-dls-text">{props.app.app_name}</div>
+ {props.compact ? null : (
+ <div className="truncate font-mono text-[11px] leading-4 text-dls-secondary" title={props.app.bundle_or_exe}>
+ {props.app.bundle_or_exe} · pid {props.app.pid}
+ </div>
+ )}
+ </div>
+ </div>
+ );
+}
+
+/** A frame with a red marker at each point (a line through them for a drag). */
+export function ComputerUseFrame(props: {
+ frame: { width: number; height: number; data_url: string };
+ points?: ComputerUsePoint[];
+ alt: string;
+ className?: string;
+}) {
+ // Percentages of the shown image, so the markers follow any displayed size.
+ const marks = scalePoints(props.points ?? [], props.frame, { width: 100, height: 100 });
+ return (
+ <div className="relative inline-block max-w-full overflow-hidden rounded-lg border border-dls-border">
+ <img src={props.frame.data_url} alt={props.alt} className={`block h-auto w-auto max-w-full ${props.className ?? "max-h-[260px]"}`} />
+ {marks.length > 1 ? (
+ <svg className="pointer-events-none absolute inset-0 size-full" viewBox="0 0 100 100" preserveAspectRatio="none" aria-hidden="true">
+ <polyline
+ points={marks.map((mark) => `${mark.x},${mark.y}`).join(" ")}
+ fill="none"
+ stroke="rgb(239 68 68)"
+ strokeWidth={2}
+ vectorEffect="non-scaling-stroke"
+ />
+ </svg>
+ ) : null}
+ {marks.map((mark, index) => (
+ <span
+ key={index}
+ aria-hidden="true"
+ className="pointer-events-none absolute size-5 -translate-x-1/2 -translate-y-1/2 rounded-full border-2 border-red-500 bg-red-500/20 shadow-[0_0_0_2px_rgba(255,255,255,0.8)]"
+ style={{ left: `${mark.x}%`, top: `${mark.y}%` }}
+ >
+ <span className="absolute left-1/2 top-1/2 size-1 -translate-x-1/2 -translate-y-1/2 rounded-full bg-red-500" />
+ </span>
+ ))}
+ </div>
+ );
+}
+
+function AgentSays(props: { text: string }) {
+ if (!props.text.trim()) return null;
+ return (
+ <blockquote className="border-l-2 border-dls-border pl-3 text-[12px] leading-5 text-dls-text">
+ <span className="text-dls-secondary">The agent says: </span>
+ {props.text}
+ </blockquote>
+ );
+}
+
+/** An approval only the person answers: open a window, or one action in it. */
+export function ComputerUseRequestCard(props: { request: ComputerUseRequest }) {
+ const request = props.request;
+ const [windowId, setWindowId] = useState(request.kind === "open" ? request.window_id : "");
+ const [busy, setBusy] = useState(false);
+ const [error, setError] = useState<string | null>(null);
+
+ const answer = (decision: ComputerUseAnswer["decision"]) => {
+ setBusy(true);
+ setError(null);
+ answerComputerUse({ request_id: request.request_id, decision, ...(request.kind === "open" ? { window_id: windowId } : {}) })
+ .catch((reason: unknown) => {
+ const message = reason instanceof Error ? stripRemoteMethodErrorPrefix(reason.message) : "";
+ setError(message || APPROVE_WITH_OWN_INPUT);
+ })
+ .finally(() => setBusy(false));
+ };
+
+ const errorLine = error ? (
+ <p role="alert" className="text-[12px] font-medium leading-5 text-red-11">{error}</p>
+ ) : null;
+
+ if (request.kind === "open") {
+ return (
+ <div data-computer-use-request={request.request_id} className="flex flex-col gap-3 border-b border-dls-border px-4 py-3">
+ <div className="flex items-center gap-2 text-[13px] font-medium leading-5 text-dls-text">
+ <MonitorSmartphone size={16} className="text-dls-secondary" />
+ Let the agent use this app?
+ </div>
+ <ComputerUseAppIdentity app={request.app} icon={request.icon} />
+ <div className="grid gap-2 text-[12px] leading-5 text-dls-text sm:grid-cols-2">
+ <label className="flex min-w-0 flex-col gap-1">
+ <span className="text-dls-secondary">Window</span>
+ {request.windows.length > 1 ? (
+ <select
+ aria-label="Window to allow"
+ className="w-full rounded-lg border border-dls-border bg-transparent px-2 py-1.5 text-[12px]"
+ value={windowId}
+ onChange={(event) => setWindowId(event.target.value)}
+ >
+ {request.windows.map((item) => (
+ <option key={item.id} value={item.id}>{item.title || "Untitled window"}</option>
+ ))}
+ </select>
+ ) : (
+ <span className="truncate">{request.windows.find((item) => item.id === request.window_id)?.title || "Untitled window"}</span>
+ )}
+ </label>
+ <div className="flex min-w-0 flex-col gap-1">
+ <span className="text-dls-secondary">Access</span>
+ <span>{request.mode === "observe" ? "See the window" : "Use mouse and keyboard"}</span>
+ </div>
+ </div>
+ <AgentSays text={request.purpose} />
+ <p className="text-[12px] leading-5 text-dls-secondary">
+ Screenshots of this window and every action are recorded with the chat and sent to the model provider. Don't approve windows with other people's personal or regulated data.
+ </p>
+ {errorLine}
+ <div className="flex flex-wrap items-center gap-2">
+ <Button type="button" size="sm" disabled={busy} onClick={() => answer("chat")} data-computer-use-answer="chat" className="h-auto py-1.5">
+ <Check data-icon="inline-start" />
+ <span className="flex flex-col items-start leading-4">
+ <span>Allow for this chat</span>
+ <span className="text-[10px] font-normal opacity-80">Same app, up to {Math.round(request.chat_seconds / 60)} minutes</span>
+ </span>
+ </Button>
+ <Button type="button" variant="outline" size="sm" disabled={busy} onClick={() => answer("once")} data-computer-use-answer="once">
+ <Clock3 data-icon="inline-start" />
+ Allow once
+ </Button>
+ <Button type="button" variant="destructive" size="sm" disabled={busy} onClick={() => answer("deny")} data-computer-use-answer="deny">
+ <XCircle data-icon="inline-start" />
+ Deny
+ </Button>
+ </div>
+ </div>
+ );
+ }
+
+ const text = typedText(request.action);
+ return (
+ <div data-computer-use-request={request.request_id} className="flex flex-col gap-3 border-b border-dls-border px-4 py-3">
+ <div className="text-[13px] font-medium leading-5 text-dls-text">Allow this action?</div>
+ <div className="flex min-w-0 items-center gap-2">
+ <ComputerUseAppIdentity app={request.app} icon={request.icon} compact />
+ {request.window_title ? <span className="truncate text-[12px] text-dls-secondary">{request.window_title}</span> : null}
+ </div>
+ {text === null ? (
+ <div className="text-[13px] font-medium leading-5 text-dls-text">{describeComputerUseAction(request.action)}</div>
+ ) : (
+ <div className="flex flex-col gap-1 text-[13px] font-medium leading-5 text-dls-text">
+ Type
+ <pre className="max-h-40 overflow-auto whitespace-pre-wrap break-words rounded-lg border border-dls-border bg-dls-hover/55 px-2.5 py-1.5 font-mono text-[12px] font-normal">{text}</pre>
+ </div>
+ )}
+ <AgentSays text={request.intent} />
+ {request.reasons.length ? (
+ <ul className="list-disc pl-5 text-[12px] leading-5 text-dls-secondary">
+ {request.reasons.map((reason, index) => <li key={`${reason.code}-${index}`}>{reason.text}</li>)}
+ </ul>
+ ) : null}
+ <ComputerUseFrame frame={request.frame} points={request.points} alt={`${request.app.app_name} window`} />
+ {errorLine}
+ <div className="flex flex-wrap items-center gap-2">
+ <Button type="button" size="sm" disabled={busy} onClick={() => answer("allow")} data-computer-use-answer="allow">
+ <Check data-icon="inline-start" />
+ Allow
+ </Button>
+ <Button type="button" variant="destructive" size="sm" disabled={busy} onClick={() => answer("deny")} data-computer-use-answer="deny">
+ <XCircle data-icon="inline-start" />
+ Deny
+ </Button>
+ </div>
+ </div>
+ );
+}
diff --git a/apps/app/src/react-app/domains/session/surface/computer-use/status-store.ts b/apps/app/src/react-app/domains/session/surface/computer-use/status-store.ts
new file mode 100644
index 00000000..ec4fdb87
--- /dev/null
+++ b/apps/app/src/react-app/domains/session/surface/computer-use/status-store.ts
@@ -0,0 +1,88 @@
+// Computer use status from Electron main: one read on first use, then the
+// pushes on "omnirush:computer-use:status" (every request, answer, session
+// change and new frame).
+
+import { useEffect, useSyncExternalStore } from "react";
+import { create } from "zustand";
+
+import type { ComputerUseAnswer, ComputerUseControlAction, ComputerUseStatus } from "@omnirush/types/desktop-ipc";
+
+let current: ComputerUseStatus | null = null;
+const listeners = new Set<() => void>();
+let stop: (() => void) | null = null;
+
+function emit(next: ComputerUseStatus) {
+ current = next;
+ for (const listener of listeners) listener();
+}
+
+function start() {
+ const bridge = window.__OMNIRUSH_ELECTRON__;
+ const invoke = bridge?.invokeDesktop;
+ if (!invoke) return;
+ let live = true;
+ let pushed = false;
+ const off = bridge?.computerUse?.onStatus?.((status) => {
+ pushed = true;
+ emit(status);
+ });
+ invoke("getComputerUseState")
+ .then((status) => {
+ if (live && !pushed) emit(status);
+ })
+ .catch(() => undefined);
+ stop = () => {
+ live = false;
+ off?.();
+ };
+}
+
+function subscribe(listener: () => void) {
+ listeners.add(listener);
+ if (listeners.size === 1) start();
+ return () => {
+ listeners.delete(listener);
+ if (listeners.size > 0) return;
+ stop?.();
+ stop = null;
+ };
+}
+
+function snapshot() {
+ return current;
+}
+
+/** The latest status, or null before the first answer (or outside the desktop app). */
+export function useComputerUseStatus(): ComputerUseStatus | null {
+ return useSyncExternalStore(subscribe, snapshot, () => null);
+}
+
+/** Chats whose surface shows computer use inline (one entry per mounted surface). */
+export const useInlineComputerUseChats = create<{ chats: string[] }>(() => ({ chats: [] }));
+
+/** Registers a chat surface that shows computer use inline, so the global fallback skips its requests. */
+export function useRegisterInlineComputerUse(sessionId: string) {
+ useEffect(() => {
+ useInlineComputerUseChats.setState((state) => ({ chats: [...state.chats, sessionId] }));
+ return () => {
+ useInlineComputerUseChats.setState((state) => {
+ const index = state.chats.indexOf(sessionId);
+ return { chats: index === -1 ? state.chats : state.chats.filter((_, at) => at !== index) };
+ });
+ };
+ }, [sessionId]);
+}
+
+/** Answers a request; rejects with main's message when the answer is refused. */
+export async function answerComputerUse(answer: ComputerUseAnswer) {
+ const invoke = window.__OMNIRUSH_ELECTRON__?.invokeDesktop;
+ if (!invoke) throw new Error("Computer use needs the desktop app.");
+ await invoke("computerUseAnswer", answer);
+}
+
+/** Stop, take over or continue a session. */
+export async function controlComputerUse(session_id: string, action: ComputerUseControlAction) {
+ const invoke = window.__OMNIRUSH_ELECTRON__?.invokeDesktop;
+ if (!invoke) throw new Error("Computer use needs the desktop app.");
+ await invoke("computerUseControl", { session_id, action });
+}
diff --git a/apps/app/src/react-app/domains/session/surface/session-surface.tsx b/apps/app/src/react-app/domains/session/surface/session-surface.tsx
index 52e83e06..0b1411a0 100644
--- a/apps/app/src/react-app/domains/session/surface/session-surface.tsx
+++ b/apps/app/src/react-app/domains/session/surface/session-surface.tsx
@@ -99,6 +99,7 @@ import { useSessionFindStore } from "./find-store";
import { getSessionActivityStatusLabel, useSessionActivityStore, type SessionActivityStatus } from "@/react-app/domains/session/status/session-activity-store";
import { PermissionApprovalPanel } from "@/react-app/domains/session/chat/permission-approval-modal";
import { QuestionPanel } from "@/react-app/domains/session/modals/question-modal";
+import { ComputerUseInline, useComputerUseForChat } from "@/react-app/domains/session/surface/computer-use/inline";
import { QueuedMessagesPanel } from "@/react-app/domains/session/modals/queued-messages-panel";
import { GoodSessionChecklistBar, noteUserStop } from "@/react-app/domains/quality/good-session";
import { PlacementTurnFooter } from "@/react-app/domains/placements/placements";
@@ -1173,6 +1174,8 @@ export function SessionSurface(props: SessionSurfaceProps) {
const findAppliedQuery = useSessionFindStore((state) => state.appliedQuery);
const setFindLastFocused = useSessionFindStore((state) => state.setLastFocused);
const findOwned = findOpen && findSessionId === props.sessionId;
+ const computerUse = useComputerUseForChat(props.sessionId);
+ const hasComputerUse = computerUse.requests.length > 0 || computerUse.sessions.length > 0;
const findHighlightQuery = findOwned && findAppliedQuery.trim().length >= 2 ? findAppliedQuery : "";
const sessionActivityStatus = useSessionActivityStore(
(state) => state.statusesByWorkspaceId[props.workspaceId]?.[props.sessionId] ?? "idle",
@@ -3610,9 +3613,9 @@ export function SessionSurface(props: SessionSurfaceProps) {
isRemoteWorkspace={props.isRemoteWorkspace}
isSandboxWorkspace={props.isSandboxWorkspace}
onUploadInboxFiles={props.onUploadInboxFiles ?? handleUploadInboxFiles}
- compactTopSpacing={Boolean(showCompactionProgress || props.activeQuestion || (props.todos ?? []).some((todo) => todo.content.trim()) || props.activePermission || queuedItems.length > 0)}
+ compactTopSpacing={Boolean(showCompactionProgress || props.activeQuestion || (props.todos ?? []).some((todo) => todo.content.trim()) || props.activePermission || queuedItems.length > 0 || hasComputerUse)}
topAccessory={
- showCompactionProgress || props.activeQuestion || (props.todos ?? []).some((todo) => todo.content.trim()) || props.activePermission || queuedItems.length > 0 ? (
+ showCompactionProgress || props.activeQuestion || (props.todos ?? []).some((todo) => todo.content.trim()) || props.activePermission || queuedItems.length > 0 || hasComputerUse ? (
<div>
{showCompactionProgress ? <ContextCompactionProgress /> : null}
{queuedItems.length > 0 ? (
@@ -3638,6 +3641,7 @@ export function SessionSurface(props: SessionSurfaceProps) {
) : (props.todos ?? []).some((todo) => todo.content.trim()) ? (
<TodoPanel todos={props.todos ?? []} />
) : null}
+ {hasComputerUse ? <ComputerUseInline requests={computerUse.requests} sessions={computerUse.sessions} /> : null}
{props.activePermission ? (
<PermissionApprovalPanel
permission={props.activePermission}
diff --git a/apps/app/tests/computer-use-describe.test.ts b/apps/app/tests/computer-use-describe.test.ts
new file mode 100644
index 00000000..9dc49632
--- /dev/null
+++ b/apps/app/tests/computer-use-describe.test.ts
@@ -0,0 +1,88 @@
+import { describe, expect, test } from "bun:test";
+import type { DynamicToolUIPart, UIMessage } from "ai";
+
+import { getAssistantRenderGroups } from "../src/components/chat/utils";
+import {
+ belongsToChat,
+ describeComputerUseAction,
+ formatMinutesLeft,
+ pointsOf,
+ scalePoints,
+} from "../src/react-app/domains/session/surface/computer-use/describe";
+
+describe("computer use action in plain words", () => {
+ test("pointer actions", () => {
+ expect(describeComputerUseAction({ type: "click", x: 120.4, y: 48 })).toBe("Click at 120, 48");
+ expect(describeComputerUseAction({ type: "double_click", x: 1, y: 2 })).toBe("Double click at 1, 2");
+ expect(describeComputerUseAction({ type: "right_click", x: 1, y: 2, modifiers: ["shift"] })).toBe("Right click at 1, 2 holding Shift");
+ expect(describeComputerUseAction({ type: "move", x: 5, y: 6 })).toBe("Move the pointer at 5, 6");
+ });
+
+ test("drag, scroll, type, key and wait", () => {
+ expect(describeComputerUseAction({ type: "drag", path: [{ x: 10, y: 10 }, { x: 50, y: 20 }, { x: 200, y: 40 }] })).toBe("Drag from 10, 10 to 200, 40");
+ expect(describeComputerUseAction({ type: "scroll", x: 400, y: 300, delta: 3 })).toBe("Scroll down 3 steps at 400, 300");
+ expect(describeComputerUseAction({ type: "scroll", x: 4, y: 3, delta: -1, axis: "horizontal" })).toBe("Scroll left 1 step at 4, 3");
+ expect(describeComputerUseAction({ type: "type", text: "hello world" })).toBe('Type "hello world"');
+ expect(describeComputerUseAction({ type: "key", key: "v", modifiers: ["ctrl"] })).toBe("Press Ctrl+V");
+ expect(describeComputerUseAction({ type: "key", key: "Enter" })).toBe("Press Enter");
+ expect(describeComputerUseAction({ type: "wait", ms: 500 })).toBe("Wait 500 ms");
+ });
+
+ test("anything else stays generic", () => {
+ expect(describeComputerUseAction(null)).toBe("Do an action");
+ expect(describeComputerUseAction({ type: "launch" })).toBe("Do an action");
+ });
+});
+
+describe("computer use points", () => {
+ test("a drag's path or a single point", () => {
+ expect(pointsOf({ type: "drag", path: [{ x: 1, y: 2 }, { x: "3", y: 4 }, { x: 5, y: 6 }] })).toEqual([{ x: 1, y: 2 }, { x: 5, y: 6 }]);
+ expect(pointsOf({ type: "click", x: 7, y: 8 })).toEqual([{ x: 7, y: 8 }]);
+ expect(pointsOf({ type: "type", text: "a" })).toEqual([]);
+ });
+
+ test("scales frame pixels to the shown size", () => {
+ expect(scalePoints([{ x: 640, y: 400 }, { x: 0, y: 800 }], { width: 1280, height: 800 }, { width: 320, height: 200 })).toEqual([
+ { x: 160, y: 100 },
+ { x: 0, y: 200 },
+ ]);
+ expect(scalePoints([{ x: 320, y: 200 }], { width: 1280, height: 800 }, { width: 100, height: 100 })).toEqual([{ x: 25, y: 25 }]);
+ expect(scalePoints([{ x: 1, y: 1 }], { width: 0, height: 800 }, { width: 100, height: 100 })).toEqual([]);
+ });
+});
+
+describe("computer use chat scope and time", () => {
+ test("no chat shows everywhere, a chat only in itself", () => {
+ expect(belongsToChat(null, "ses_a")).toBe(true);
+ expect(belongsToChat("ses_a", "ses_a")).toBe(true);
+ expect(belongsToChat("ses_b", "ses_a")).toBe(false);
+ });
+
+ test("minutes left", () => {
+ expect(formatMinutesLeft(30)).toBe("less than a minute");
+ expect(formatMinutesLeft(605)).toBe("10 min");
+ });
+});
+
+describe("computer use screenshots in the transcript", () => {
+ const act: DynamicToolUIPart = {
+ type: "dynamic-tool",
+ toolName: "computer-use_computer_act",
+ toolCallId: "call_1",
+ state: "output-available",
+ input: { intent: "Open the menu", action: { type: "click", x: 1, y: 1 } },
+ output: '{"ok":true}',
+ };
+ const shot: UIMessage["parts"][number] = { type: "file", mediaType: "image/png", url: "data:image/png;base64,AAAA" };
+
+ test("an image right after a computer use call joins its tool line", () => {
+ const groups = getAssistantRenderGroups([act, shot], false);
+ expect(groups).toHaveLength(1);
+ expect(groups[0]).toMatchObject({ kind: "tool", attachments: [shot] });
+ });
+
+ test("other tools keep their files as separate groups", () => {
+ const other: DynamicToolUIPart = { ...act, toolName: "github_search" };
+ expect(getAssistantRenderGroups([other, shot], false).map((group) => group.kind)).toEqual(["tool", "file"]);
+ });
+});
diff --git a/apps/app/tests/extension-taxonomy.test.ts b/apps/app/tests/extension-taxonomy.test.ts
index 3cd7e109..600efa89 100644
--- a/apps/app/tests/extension-taxonomy.test.ts
+++ b/apps/app/tests/extension-taxonomy.test.ts
@@ -1,6 +1,6 @@
import { describe, expect, test } from "bun:test";
-import { COMPUTER_USE_PAUSED, MCP_QUICK_CONNECT, type McpDirectoryInfo } from "../src/app/constants";
+import { COMPUTER_USE_LIBRARY_CONNECT, MCP_QUICK_CONNECT, type McpDirectoryInfo } from "../src/app/constants";
import {
matchesExtensionFilter,
taxonomyForDirectoryEntry,
@@ -14,13 +14,13 @@ function builtInEntry(id: string): McpDirectoryInfo {
describe("extension taxonomy", () => {
test("built-ins are apps because they run on this device", () => {
- for (const id of ["omnirush-browser", ...(COMPUTER_USE_PAUSED ? [] : ["computer-use"]), "ollama"]) {
+ for (const id of ["omnirush-browser", ...(COMPUTER_USE_LIBRARY_CONNECT ? ["computer-use"] : []), "ollama"]) {
expect(taxonomyForDirectoryEntry(builtInEntry(id))).toBe("app");
}
});
test("Computer Use is not offered while it is paused", () => {
- expect(MCP_QUICK_CONNECT.some((entry) => entry.id === "computer-use")).toBe(!COMPUTER_USE_PAUSED);
+ expect(MCP_QUICK_CONNECT.some((entry) => entry.id === "computer-use")).toBe(COMPUTER_USE_LIBRARY_CONNECT);
});
test("Google Workspace is not a built-in app; it arrives as an org connection", () => {
diff --git a/apps/app/tests/mcp-local-server-recovery.test.ts b/apps/app/tests/mcp-local-server-recovery.test.ts
index 8e4e6123..cf14759b 100644
--- a/apps/app/tests/mcp-local-server-recovery.test.ts
+++ b/apps/app/tests/mcp-local-server-recovery.test.ts
@@ -1,6 +1,6 @@
import { afterEach, describe, expect, test } from "bun:test";
-import { COMPUTER_USE_PAUSED, MCP_QUICK_CONNECT } from "../src/app/constants";
+import { COMPUTER_USE_LIBRARY_CONNECT, MCP_QUICK_CONNECT } from "../src/app/constants";
import { createOmniRushServerClient } from "../src/app/lib/omnirush-server";
import { createOmniRushServerStore } from "../src/react-app/domains/connections/omnirush-server-store";
import type { McpDirectoryInfo } from "../src/app/constants";
@@ -76,7 +76,7 @@ describe("local MCP server recovery", () => {
});
// The setup flow is not reachable while Computer Use is paused (no catalog entry).
-describe.skipIf(COMPUTER_USE_PAUSED)("bundled Computer Use setup", () => {
+describe.skipIf(!COMPUTER_USE_LIBRARY_CONNECT)("bundled Computer Use setup", () => {
async function connectWithHelper(command: string[] | null) {
Object.defineProperty(globalThis, "window", {
configurable: true,
diff --git a/apps/desktop/electron/computer-use-host.mjs b/apps/desktop/electron/computer-use-host.mjs
index a8bc9773..6e91b1cf 100644
--- a/apps/desktop/electron/computer-use-host.mjs
+++ b/apps/desktop/electron/computer-use-host.mjs
@@ -1,77 +1,180 @@
-// The engine sees only MCP. Only the main-window IPC handler can send UI actions.
-import { createServer } from "node:net";
+// macOS computer use host (contract v2). The tool surface, validation,
+// results, per-action approvals, `computer.session` events, media spool and
+// MCP transport are the shared portable host's
+// (apps/server/src/computer-use/shared/portable-host.mjs), exactly as on
+// Windows and Linux; the signed Swift helper (`ComputerUse worker`,
+// packages/computer-use/native NativeWorker.swift) is its native layer:
+// window list, capture (with password-field boxes and on-device text
+// recognition), hit testing, window-targeted input and person-input pauses.
+//
+// createComputerUseHost() returns the same interface as createPortableHost()
+// (availability, os, connect, listen(dir) → { discoveryPath, close }, control,
+// stopAll, syntheticInput, watchInput, state, close) and calls the same
+// callbacks (approve.open / approve.act, onEvent, onState, onFrame), so
+// Electron main treats both hosts alike. The engine reaches it through the
+// bundled relay with the discovery file, as on the other platforms.
import { spawn } from "node:child_process";
-import { createHash, randomUUID } from "node:crypto";
-import { mkdir, lstat, chmod, unlink } from "node:fs/promises";
-import path from "node:path";
-import os from "node:os";
-const engineMethods = new Set(["initialize", "ping", "tools/list", "tools/call", "notifications/initialized", "notifications/cancelled"]);
-export async function createComputerUseHost({ profile, executable }) {
- const directory = path.join(os.tmpdir(), `ow-computer-${createHash("sha256").update(profile).digest("hex").slice(0, 16)}`);
- await mkdir(directory, { mode: 0o700, recursive: true });
- const stat = await lstat(directory);
- if (!stat.isDirectory() || stat.uid !== process.getuid() || (stat.mode & 0o077)) throw new Error("Computer Use control directory is not private.");
- const socketPath = path.join(directory, "host.sock");
- // Only this profile's main process owns this endpoint. A stale socket is left
- // after a crash; a live profile is already protected by Electron's instance lock.
- await unlink(socketPath).catch((error) => { if (error.code !== "ENOENT") throw error; });
- const connections = new Map();
- const server = createServer((socket) => {
- if (connections.size >= 16) { socket.destroy(); return; }
- const connectionId = randomUUID();
- const child = spawn(executable, ["mcp-hosted"], { stdio: ["pipe", "pipe", "ignore"] });
- const entry = { child, state: null }; connections.set(connectionId, entry);
- let incoming = "", outgoing = "", closed = false;
- const close = () => {
- if (closed) return; closed = true; connections.delete(connectionId);
- socket.destroy(); child.stdin.end(); child.kill("SIGTERM");
- const timeout = setTimeout(() => { if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); }, 2_000);
- timeout.unref(); child.once("exit", () => clearTimeout(timeout));
- };
- entry.close = close;
- socket.setEncoding("utf8"); child.stdout.setEncoding("utf8");
- socket.on("end", close); socket.on("error", close); socket.on("close", close); child.on("error", close); child.on("exit", close);
- child.stdin.on("error", close);
- socket.on("data", (chunk) => {
- incoming += chunk;
- if (incoming.length > 1_048_576) { close(); return; }
- let end;
- while ((end = incoming.indexOf("\n")) >= 0) {
- const line = incoming.slice(0, end); incoming = incoming.slice(end + 1);
- let message; try { message = JSON.parse(line); } catch { close(); return; }
- if (!message || message.jsonrpc !== "2.0" || !engineMethods.has(message.method)) {
- socket.write(`${JSON.stringify({ jsonrpc: "2.0", id: message?.id ?? null, error: { code: -32601, message: "Method not available to the agent." } })}\n`);
- continue;
- }
- child.stdin.write(`${line}\n`);
+const SHARED = new URL("../../server/src/computer-use/shared/", import.meta.url);
+const record = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+const MAX_LINE = 1_048_576;
+const MAX_FRAME = 7680 * 7680 * 4;
+
+/** Boxes (capture pixels) of recognized text that looks like a credential: the matching words, else the whole line. */
+export function credentialBoxes(lines, looksLikeCredential) {
+ const boxes = [];
+ const box = (b) => (record(b) && [b.x, b.y, b.width, b.height].every(Number.isFinite) ? { x: b.x - 2, y: b.y - 2, width: b.width + 4, height: b.height + 4 } : null);
+ for (const line of Array.isArray(lines) ? lines : []) {
+ if (!record(line) || typeof line.text !== "string") continue;
+ const words = (Array.isArray(line.words) ? line.words : []).filter((w) => record(w) && typeof w.text === "string");
+ const hits = words.filter((w) => looksLikeCredential(w.text));
+ // `password: hunter22` matches as a line but not word by word: the words after the label go too.
+ const pairs = words.slice(1).filter((w, i) => !hits.includes(w) && looksLikeCredential(`${words[i].text} ${w.text}`));
+ const picked = [...hits, ...pairs].map((w) => box(w.box)).filter(Boolean);
+ if (picked.length) boxes.push(...picked);
+ else if (looksLikeCredential(line.text)) { const b = box(line.box); if (b) boxes.push(b); }
+ }
+ return boxes;
+}
+
+/**
+ * The native worker over the Swift helper's stdio (the NativeWorker interface
+ * of portable-host.d.mts). `captureLongSide()` caps capture size (the
+ * full_long_side limit); `looksLikeCredential` (policy.mjs) picks the
+ * recognized words that are painted over.
+ */
+export function createMacNativeWorker({ executable, args = ["worker"], captureLongSide = () => 3840, looksLikeCredential = () => false, paintBoxes = null, timeoutMs = 10_000 }) {
+ const child = spawn(executable, args, { stdio: ["pipe", "pipe", "ignore"] });
+ const pending = new Map();
+ const listeners = new Set();
+ let nextId = 0, closed = false, held = { keys: [], buttons: [] }, release = null, lastOcr = "none";
+ let readyResolve;
+ const ready = new Promise((resolve) => { readyResolve = resolve; });
+ const readyTimer = setTimeout(() => readyResolve({ supported: false, code: "native_unavailable", error: "Desktop access did not start. Restart the app." }), timeoutMs);
+
+ const deliver = (value) => {
+ if (!record(value)) return;
+ if (value.event === "ready") { clearTimeout(readyTimer); readyResolve({ supported: value.supported === true, platform: "darwin", ...(value.code ? { code: value.code } : {}), ...(value.error ? { error: value.error } : {}) }); return; }
+ if (value.event === "held") { held = { keys: value.keys ?? [], buttons: value.buttons ?? [] }; release = record(value.release) ? value.release : null; return; }
+ if (value.event) { for (const listener of listeners) listener(value); return; }
+ const request = pending.get(value.id);
+ if (!request) return;
+ pending.delete(value.id); clearTimeout(request.timer);
+ if (record(value.error)) request.reject(Object.assign(new Error(String(value.error.message)), { code: value.error.code }));
+ else request.resolve(value.result);
+ };
+
+ // JSON lines; a header with result.bytes is followed by that many raw bytes.
+ let line = [], lineBytes = 0, binary = null;
+ child.stdout.on("data", (chunk) => {
+ let offset = 0;
+ while (offset < chunk.length) {
+ if (binary) {
+ const take = Math.min(binary.need - binary.have, chunk.length - offset);
+ binary.parts.push(chunk.subarray(offset, offset + take));
+ binary.have += take; offset += take;
+ if (binary.have === binary.need) { const { message, parts, need } = binary; binary = null; message.result.data = Buffer.concat(parts, need); deliver(message); }
+ continue;
}
- });
- child.stdout.on("data", (chunk) => {
- outgoing += chunk;
- if (outgoing.length > 16 * 1024 * 1024) { close(); return; }
- let end;
- while ((end = outgoing.indexOf("\n")) >= 0) {
- const line = outgoing.slice(0, end); outgoing = outgoing.slice(end + 1);
- let message; try { message = JSON.parse(line); } catch { close(); return; }
- if (message.method === "omnirush/ui") {
- if (message.params?.phase === "closed") entry.state = null;
- else if (message.params && typeof message.params.id === "string") entry.state = { ...entry.state, ...message.params, connectionId, helperPid: child.pid };
- } else socket.write(`${line}\n`);
+ const end = chunk.indexOf(10, offset);
+ if (end < 0) {
+ line.push(chunk.subarray(offset)); lineBytes += chunk.length - offset;
+ if (lineBytes > MAX_LINE) child.kill();
+ return;
}
- });
+ line.push(chunk.subarray(offset, end));
+ const text = Buffer.concat(line).toString("utf8");
+ line = []; lineBytes = 0; offset = end + 1;
+ let message;
+ try { message = JSON.parse(text); } catch { continue; }
+ const bytes = message?.result?.bytes;
+ if (record(message?.result) && Number.isSafeInteger(bytes)) {
+ if (bytes < 0 || bytes > MAX_FRAME) { child.kill(); return; }
+ delete message.result.bytes;
+ if (bytes === 0) { message.result.data = Buffer.alloc(0); deliver(message); }
+ else binary = { message, need: bytes, have: 0, parts: [] };
+ continue;
+ }
+ deliver(message);
+ }
});
- await new Promise((resolve, reject) => { server.once("error", reject); server.listen(socketPath, () => resolve(undefined)); });
- await chmod(socketPath, 0o600);
+
+ const fail = (error) => {
+ if (closed) return;
+ clearTimeout(readyTimer); readyResolve({ supported: false, code: "native_unavailable", error: error.message });
+ closed = true;
+ for (const request of pending.values()) { clearTimeout(request.timer); request.reject(error); }
+ pending.clear();
+ if (held.buttons.length && release) {
+ // The helper died with a button pressed: lift it from a fresh one.
+ const releaser = spawn(executable, ["release", JSON.stringify(release)], { stdio: "ignore" });
+ releaser.on("error", () => {});
+ held = { keys: [], buttons: [] }; release = null;
+ }
+ for (const listener of listeners) listener({ event: "unavailable" });
+ };
+ child.on("error", fail);
+ child.on("exit", () => fail(new Error("Desktop access stopped. Restart the app.")));
+ child.stdin.on("error", () => {});
+
+ const raw = (method, params = {}, ms = timeoutMs) => {
+ if (closed) return Promise.reject(Object.assign(new Error("Desktop access has stopped."), { code: "native_unavailable" }));
+ const id = ++nextId;
+ return new Promise((resolve, reject) => {
+ const timer = setTimeout(() => { pending.delete(id); reject(Object.assign(new Error("Desktop operation timed out."), { code: "timeout" })); }, ms);
+ pending.set(id, { resolve, reject, timer });
+ child.stdin.write(`${JSON.stringify({ id, method, params })}\n`);
+ });
+ };
+
return {
- socketPath,
- state: () => [...connections.values()].flatMap((entry) => entry.state ? [entry.state] : []),
- action(value) {
- if (!value || typeof value.connectionId !== "string" || typeof value.id !== "string" || !["approve", "deny", "resume", "stop", "hide", "show"].includes(value.action)) throw new Error("Invalid Computer Use control.");
- const entry = connections.get(value.connectionId);
- if (!entry?.state || entry.state.id !== value.id) throw new Error("This Computer Use request has ended.");
- entry.child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", method: "omnirush/ui", params: { id: value.id, action: value.action, windowId: value.windowId } })}\n`);
+ ready,
+ pid: child.pid,
+ get held() { return held; },
+ /** Whether on-device text recognition ran on the latest capture ("client" | "none"). */
+ get ocr() { return lastOcr; },
+ onEvent(listener) { listeners.add(listener); return () => listeners.delete(listener); },
+ async call(method, params = {}, ms = timeoutMs) {
+ if (method !== "capture") return raw(method, params, ms);
+ const frame = await raw(method, { ...params, max_long_side: captureLongSide() }, ms);
+ if (!record(frame) || !Number.isInteger(frame.width) || !Number.isInteger(frame.height) || !Buffer.isBuffer(frame.data) || frame.data.length !== frame.width * frame.height * 4) {
+ throw Object.assign(new Error("The window image could not be read."), { code: "capture_unavailable" });
+ }
+ // Password fields (accessibility) plus recognized text that looks like a credential: painted on both copies.
+ const text = credentialBoxes(frame.text_boxes, looksLikeCredential);
+ lastOcr = frame.ocr === "client" ? "client" : "none";
+ const secure = [...(Array.isArray(frame.secure) ? frame.secure : []), ...text];
+ // Painted here on every capture (the host paints the same boxes again, which changes nothing), so
+ // the host's stale check compares a masked observation with a masked fresh capture.
+ paintBoxes?.(frame, secure);
+ return { width: frame.width, height: frame.height, data: frame.data, secure, caret: frame.caret ?? null, ocr: frame.ocr === "client" ? "client" : "none", credential_boxes: text.length, source: frame.source };
},
- close() { for (const entry of connections.values()) entry.close(); server.close(); },
+ kill() { child.kill(); },
+ async close() { if (closed) return; await raw("shutdown", {}, 3000).catch(() => {}); closed = true; if (child.exitCode === null) child.kill(); },
};
}
+
+/**
+ * The macOS host. Options as createPortableHost's (spool, settings, context,
+ * approve, onEvent, onState, onFrame) plus `executable`, the helper binary
+ * (OmniRush.ai Computer Use.app/Contents/MacOS/ComputerUse). `shared` may
+ * supply the shared modules ({ portable, policy, contract, image }) when they are
+ * bundled elsewhere.
+ */
+export async function createComputerUseHost({ executable, spool = null, settings, context, approve, onEvent, onState, onFrame, shared = {} }) {
+ if (!executable) throw new Error("Computer Use helper unavailable.");
+ const portable = shared.portable ?? await import(new URL("portable-host.mjs", SHARED).href);
+ const policy = shared.policy ?? await import(new URL("policy.mjs", SHARED).href);
+ const contract = shared.contract ?? await import(new URL("contract.mjs", SHARED).href);
+ const image = shared.image ?? await import(new URL("image.mjs", SHARED).href);
+ const native = createMacNativeWorker({
+ executable,
+ captureLongSide: () => contract.normalizeLimits(settings?.()?.limits).full_long_side,
+ looksLikeCredential: policy.looksLikeCredential,
+ paintBoxes: image.paintBoxes,
+ });
+ // The archive copy records that the frame was read on the device (text recognition ran).
+ const ocrSpool = spool ? Object.create(spool, { put: { value: (frame) => spool.put({ ...frame, ocr: native.ocr }) } }) : null;
+ const host = await portable.createPortableHost({ native, spool: ocrSpool, settings, context, approve, onEvent, onState, onFrame, platform: "darwin" });
+ return { ...host, executable };
+}
diff --git a/apps/desktop/electron/computer-use-native-worker.mjs b/apps/desktop/electron/computer-use-native-worker.mjs
deleted file mode 100644
index daaf83c9..00000000
--- a/apps/desktop/electron/computer-use-native-worker.mjs
+++ /dev/null
@@ -1,490 +0,0 @@
-// Native desktop access lives off the Electron main thread. No shell execution.
-const parentPort = { postMessage: (value) => process.send?.(value), on: (name, listener) => process.on(name === "close" ? "disconnect" : name, listener), close: () => process.disconnect() };
-import { readFileSync, readlinkSync } from "node:fs";
-import path from "node:path";
-import koffi from "koffi";
-import { crc32, deflateSync } from "node:zlib";
-
-let generation = 0;
-let backend;
-const changed = () => { generation++; parentPort.postMessage({ event: "person-input", generation }); };
-const fail = (code, message) => { throw Object.assign(new Error(message), { code }); };
-const protectedNames = new Set(["cmd.exe", "powershell.exe", "pwsh.exe", "windowsterminal.exe", "conhost.exe", "credentialuibroker.exe", "consent.exe", "logonui.exe", "keepass.exe", "keepassxc.exe", "1password.exe", "gnome-terminal-server", "konsole", "xterm", "alacritty", "kitty", "foot", "gnome-keyring-daemon", "keepassxc", "1password"]);
-function safeWindow(value) {
- if ((value.pid === process.pid || value.pid === process.ppid) || protectedNames.has(path.basename(value.executable).toLowerCase())) fail("protected_app", "Use a shell tool for terminals. Computer Use cannot operate the host, security prompts, or password managers.");
- if (!Number.isSafeInteger(value.id) || !Number.isSafeInteger(value.pid) || value.pid <= 0) fail("window_unavailable", "This window cannot be identified safely.");
- return value;
-}
-const keyNames = { Enter: 13, Tab: 9, Escape: 27, Backspace: 8, Delete: 46, Space: 32, Left: 37, Up: 38, Right: 39, Down: 40, Home: 36, End: 35, PageUp: 33, PageDown: 34 };
-function keys(action) {
- if (typeof action.key !== "string") fail("invalid_action", "A key name is required.");
- const modifiers = action.modifiers ?? [];
- if (!Array.isArray(modifiers) || modifiers.some((v) => !["ctrl", "shift", "alt"].includes(v)) || new Set(modifiers).size !== modifiers.length) fail("invalid_action", "Only ctrl, shift, and alt modifiers are supported.");
- if ((modifiers.includes("alt") && ["Tab", "Escape", "F4"].includes(action.key)) || (modifiers.includes("ctrl") && (modifiers.includes("alt") || action.key === "Escape"))) fail("unsupported_action", "System-wide shortcuts are unavailable.");
- if (!(action.key in keyNames) && !/^[a-z0-9]$/i.test(action.key) && !/^F([1-9]|1[0-2])$/.test(action.key)) fail("unsupported_action", "This key is not supported.");
- return modifiers;
-}
-
-function windows() {
- const u = koffi.load("user32.dll"), k = koffi.load("kernel32.dll"), dwm = koffi.load("dwmapi.dll");
- const f = (lib, declaration) => lib.func(declaration);
- const rect = koffi.struct("CU_RECT", { left: "int32_t", top: "int32_t", right: "int32_t", bottom: "int32_t" });
- const point = koffi.struct("CU_POINT", { x: "int32_t", y: "int32_t" });
- const filetime = koffi.struct("CU_FILETIME", { low: "uint32_t", high: "uint32_t" });
- const mouse = koffi.struct("CU_MOUSEINPUT", { dx: "int32_t", dy: "int32_t", mouseData: "uint32_t", dwFlags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t" });
- const keyboard = koffi.struct("CU_KEYBDINPUT", { wVk: "uint16_t", wScan: "uint16_t", dwFlags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t" });
- const hardware = koffi.struct("CU_HARDWAREINPUT", { uMsg: "uint32_t", wParamL: "uint16_t", wParamH: "uint16_t" });
- const input = koffi.struct("CU_INPUT", { type: "uint32_t", value: koffi.union("CU_INPUT_VALUE", { mi: mouse, ki: keyboard, hi: hardware }) });
- const hookKey = koffi.struct("CU_KBDLLHOOKSTRUCT", { vkCode: "uint32_t", scanCode: "uint32_t", flags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t" });
- const hookMouse = koffi.struct("CU_MSLLHOOKSTRUCT", { pt: point, mouseData: "uint32_t", flags: "uint32_t", time: "uint32_t", dwExtraInfo: "uintptr_t" });
- const msg = koffi.struct("CU_MSG", { hwnd: "uintptr_t", message: "uint32_t", wParam: "uintptr_t", lParam: "intptr_t", time: "uint32_t", pt: point, lPrivate: "uint32_t" });
- const enumProto = koffi.proto("int __stdcall CU_ENUMPROC(uintptr_t hwnd, intptr_t parameter)");
- // Low-level hook LPARAM points to event data. Keep its pointer type so Koffi
- // passes an external pointer to decode rather than converting it to a number.
- const hookProto = koffi.proto("intptr_t __stdcall CU_HOOKPROC(int code, uintptr_t wParam, void *lParam)");
- const enumWindows = f(u, "int __stdcall EnumWindows(CU_ENUMPROC *callback, intptr_t parameter)");
- const visible = f(u, "int __stdcall IsWindowVisible(uintptr_t hwnd)");
- const isWindow = f(u, "int __stdcall IsWindow(uintptr_t hwnd)");
- const iconic = f(u, "int __stdcall IsIconic(uintptr_t hwnd)");
- const pidOf = f(u, "uint32_t __stdcall GetWindowThreadProcessId(uintptr_t hwnd, _Out_ uint32_t *pid)");
- const titleOf = f(u, "int __stdcall GetWindowTextW(uintptr_t hwnd, _Out_ uint16_t *text, int count)");
- const boundsOf = f(dwm, "int32_t __stdcall DwmGetWindowAttribute(uintptr_t hwnd, uint32_t attribute, _Out_ CU_RECT *rect, uint32_t size)");
- const foreground = f(u, "uintptr_t __stdcall GetForegroundWindow()");
- const ancestor = f(u, "uintptr_t __stdcall GetAncestor(uintptr_t hwnd, uint32_t flag)");
- const fromPoint = f(u, "uintptr_t __stdcall WindowFromPoint(CU_POINT point)");
- const focus = f(u, "int __stdcall SetForegroundWindow(uintptr_t hwnd)");
- const send = f(u, "uint32_t __stdcall SendInput(uint32_t count, CU_INPUT *inputs, int size)");
- const metrics = f(u, "int __stdcall GetSystemMetrics(int index)");
- const openProcess = f(k, "uintptr_t __stdcall OpenProcess(uint32_t access, int inherit, uint32_t pid)");
- const processName = f(k, "int __stdcall QueryFullProcessImageNameW(uintptr_t process, uint32_t flags, _Out_ uint16_t *name, _Inout_ uint32_t *size)");
- const times = f(k, "int __stdcall GetProcessTimes(uintptr_t process, _Out_ CU_FILETIME *created, _Out_ CU_FILETIME *exited, _Out_ CU_FILETIME *kernel, _Out_ CU_FILETIME *user)");
- const closeHandle = f(k, "int __stdcall CloseHandle(uintptr_t handle)");
- const openDesktop = f(u, "uintptr_t __stdcall OpenInputDesktop(uint32_t flags, int inherit, uint32_t access)");
- const closeDesktop = f(u, "int __stdcall CloseDesktop(uintptr_t desktop)");
- const userInfo = f(u, "int __stdcall GetUserObjectInformationW(uintptr_t object, int index, _Out_ uint16_t *value, uint32_t size, _Out_ uint32_t *needed)");
- f(u, "intptr_t __stdcall SetThreadDpiAwarenessContext(intptr_t context)")(-4);
- const marker = 0x4f4d4e49;
- const inject = (value) => { if (send(1, [value], koffi.sizeof(input)) !== 1) fail("input_unavailable", "Windows blocked input. Keep the app on the active desktop, run it without administrator privileges, and continue."); };
- const keyEvent = (vk, scan, flags) => inject({ type: 1, value: { ki: { wVk: vk, wScan: scan, dwFlags: flags, time: 0, dwExtraInfo: marker } } });
- const mouseEvent = (flags, data = 0, dx = 0, dy = 0) => inject({ type: 0, value: { mi: { dx, dy, mouseData: data >>> 0, dwFlags: flags, time: 0, dwExtraInfo: marker } } });
- const downKeys = new Set();
- let pointerDown = false;
- const release = () => {
- for (const key of downKeys) { try { keyEvent(key, 0, 2); } catch {} }
- downKeys.clear();
- if (pointerDown) { try { mouseEvent(4); } catch {} pointerDown = false; }
- };
- const peek = f(u, "int __stdcall PeekMessageW(_Out_ CU_MSG *message, uintptr_t hwnd, uint32_t first, uint32_t last, uint32_t remove)");
- const nextHook = f(u, "intptr_t __stdcall CallNextHookEx(uintptr_t hook, int code, uintptr_t wParam, void *lParam)");
- const installHook = f(u, "uintptr_t __stdcall SetWindowsHookExW(int type, CU_HOOKPROC *callback, uintptr_t module, uint32_t thread)");
- const uninstall = f(u, "int __stdcall UnhookWindowsHookEx(uintptr_t hook)");
- let callbacks = [], hooks = [];
- // Global hooks are needed only while an approved Computer Use session is active.
- // Keeping the worker alive for discovery and capture must not intercept all user input.
- const startMonitoring = () => {
- if (hooks.length) return;
- callbacks = [hookKey, hookMouse].map((type) => koffi.register((code, wParam, lParam) => {
- if (code >= 0) {
- const value = koffi.decode(lParam, type);
- if (Number(value.dwExtraInfo) !== marker) { changed(); release(); }
- }
- return nextHook(0, code, wParam, lParam);
- }, koffi.pointer(hookProto)));
- hooks = [installHook(13, callbacks[0], 0, 0), installHook(14, callbacks[1], 0, 0)];
- if (hooks.some((v) => !v)) {
- hooks.forEach((h) => { if (h) uninstall(h); });
- callbacks.forEach((c) => koffi.unregister(c));
- hooks = [];
- callbacks = [];
- fail("input_unavailable", "Windows input monitoring is unavailable. Restart OmniRush.ai on your normal desktop.");
- }
- };
- const stopMonitoring = () => {
- hooks.forEach((hook) => uninstall(hook));
- callbacks.forEach((callback) => koffi.unregister(callback));
- hooks = [];
- callbacks = [];
- };
- const poll = () => {
- if (!hooks.length) return;
- const value = {};
- while (peek(value, 0, 0, 0, 1)) {}
- };
- const available = () => {
- const handle = openDesktop(0, 0, 1);
- if (!handle) return false;
- try { const data = Buffer.alloc(512), size = [0]; return Boolean(userInfo(handle, 2, data, data.length, size)) && data.toString("utf16le").split("\0")[0].toLowerCase() === "default"; }
- finally { closeDesktop(handle); }
- };
- function window(id) {
- if (!available() || !isWindow(id) || !visible(id) || iconic(id)) fail("window_unavailable", "Keep the approved app visible on an unlocked desktop.");
- const pid = [0]; pidOf(id, pid);
- const handle = openProcess(0x1000, 0, pid[0]);
- if (!handle) fail("window_unavailable", "This app cannot be inspected. Run it without administrator privileges.");
- let executable, identity;
- try {
- const name = Buffer.alloc(65536), size = [32768], created = {};
- if (!processName(handle, 0, name, size) || !times(handle, created, {}, {}, {})) fail("window_unavailable", "The app identity could not be verified.");
- executable = name.toString("utf16le", 0, size[0] * 2);
- identity = String(pid[0]) + ":" + created.high + ":" + created.low;
- } finally { closeHandle(handle); }
- const box = {}, title = Buffer.alloc(8192);
- if (boundsOf(id, 9, box, koffi.sizeof(rect)) !== 0) fail("window_unavailable", "The window bounds could not be read.");
- titleOf(id, title, 4096);
- const bounds = { x: box.left, y: box.top, width: box.right - box.left, height: box.bottom - box.top };
- if (bounds.width <= 0 || bounds.height <= 0) fail("window_unavailable", "The window is not visible.");
- return safeWindow({ id: Number(id), pid: pid[0], identity, executable, appId: executable.toLowerCase(), appName: path.win32.basename(executable, ".exe"), title: title.toString("utf16le").split("\0")[0], bounds });
- }
- function guard(expected, expectedGeneration, pointValue, deadline) {
- if (Date.now() >= deadline) fail("session_expired", "The approved session expired. Start a new session.");
- poll();
- if (generation !== expectedGeneration) fail("user_interacting", "Your input interrupted Computer Use. Choose Continue when ready.");
- const current = window(expected.id);
- if (current.identity !== expected.identity || JSON.stringify(current.bounds) !== JSON.stringify(expected.bounds)) fail("window_changed", "The approved window changed. Observe it again.");
- if (Number(ancestor(foreground(), 2)) !== expected.id) fail("focus_changed", "The approved window must stay in front. Choose Continue when ready.");
- if (pointValue && Number(ancestor(fromPoint(pointValue), 2)) !== expected.id) fail("window_occluded", "Another window covers this point. Move it away and continue.");
- return current;
- }
- const move = (x, y) => {
- const left = metrics(76), top = metrics(77), width = metrics(78), height = metrics(79);
- if (width <= 1 || height <= 1) fail("input_unavailable", "The desktop bounds could not be read.");
- mouseEvent(0x8000 | 0x4000 | 1, 0, Math.round((x - left) * 65535 / (width - 1)), Math.round((y - top) * 65535 / (height - 1)));
- };
- return {
- poll, window, available, release, startMonitoring, stopMonitoring, check: guard,
- list() { const result = []; enumWindows((id) => { try { const item = window(id); if (item.title) result.push(item); } catch {} return 1; }, 0); return result; },
- focus(expected) { if (window(expected.id).identity !== expected.identity || !focus(expected.id)) fail("focus_changed", "Bring the approved app to the front, then choose Continue."); poll(); },
- async act(expected, expectedGeneration, action, deadline) {
- const checked = (p) => guard(expected, expectedGeneration, p, deadline);
- const pointValue = (p) => ({ x: expected.bounds.x + Math.round(p.x), y: expected.bounds.y + Math.round(p.y) });
- try {
- checked();
- if (action.type === "type") {
- for (const char of action.text) {
- checked();
- for (let i = 0; i < char.length; i++) { keyEvent(0, char.charCodeAt(i), 4); keyEvent(0, char.charCodeAt(i), 6); }
- await new Promise((resolve) => setTimeout(resolve, 2));
- }
- } else if (action.type === "key") {
- const modifiers = keys(action), vk = keyNames[action.key] ?? (/^F/.test(action.key) ? 111 + Number(action.key.slice(1)) : action.key.toUpperCase().charCodeAt(0));
- for (const modifier of modifiers) { const key = { ctrl: 17, shift: 16, alt: 18 }[modifier]; downKeys.add(key); keyEvent(key, 0, 0); }
- checked(); downKeys.add(vk); keyEvent(vk, 0, 0); keyEvent(vk, 0, 2); downKeys.delete(vk);
- } else if (["click", "double_click", "scroll", "drag"].includes(action.type)) {
- const points = action.type === "drag" ? action.path : [action];
- for (const p of points) {
- const position = pointValue(p); checked(position); move(position.x, position.y); checked(position);
- if (action.type === "drag" && !pointerDown) { pointerDown = true; mouseEvent(2); }
- await new Promise((resolve) => setTimeout(resolve, action.type === "scroll" ? 32 : action.type === "drag" ? 12 : 2));
- }
- const position = pointValue(points.at(-1)); checked(position);
- if (action.type === "scroll") mouseEvent(action.axis === "horizontal" ? 0x1000 : 0x800, action.delta * (action.axis === "horizontal" ? 120 : -120));
- else if (action.type !== "drag") {
- for (let i = 0; i < (action.type === "double_click" ? 2 : 1); i++) {
- checked(position); pointerDown = true; mouseEvent(2); mouseEvent(4); pointerDown = false;
- if (i === 0 && action.type === "double_click") await new Promise((resolve) => setTimeout(resolve, 60));
- }
- }
- } else fail("unsupported_action", "Use click, double_click, drag, scroll, type, or key.");
- } finally { release(); }
- },
- close() { release(); stopMonitoring(); },
- };
-}
-
-function x11() {
- if (process.env.XDG_SESSION_TYPE === "wayland" || process.env.WAYLAND_DISPLAY) fail("wayland_unavailable", "Wayland needs a desktop portal connection. This build supports Linux X11; sign in to an X11 session or use the built-in browser and app connectors.");
- if (!process.env.DISPLAY) fail("desktop_unavailable", "Computer Use needs a graphical Linux desktop.");
- const x = koffi.load("libX11.so.6"), xt = koffi.load("libXtst.so.6"), xi = koffi.load("libXi.so.6");
- const attributes = koffi.struct("CU_XWindowAttributes", {
- x: "int", y: "int", width: "int", height: "int", border_width: "int", depth: "int", visual: "void *", root: "ulong", class: "int", bit_gravity: "int", win_gravity: "int", backing_store: "int", backing_planes: "ulong", backing_pixel: "ulong", save_under: "int", colormap: "ulong", map_installed: "int", map_state: "int", all_event_masks: "long", your_event_mask: "long", do_not_propagate_mask: "long", override_redirect: "int", screen: "void *",
- });
- const cookieType = koffi.struct("CU_XGenericEventCookie", { type: "int", serial: "ulong", send_event: "int", display: "void *", extension: "int", evtype: "int", cookie: "uint", data: "void *" });
- const rawType = koffi.struct("CU_XIRawEvent", { type: "int", serial: "ulong", send_event: "int", display: "void *", extension: "int", evtype: "int", time: "ulong", deviceid: "int", sourceid: "int", detail: "int", flags: "int", valuators: koffi.struct({ mask_len: "int", mask: "void *", values: "void *" }), raw_values: "void *" });
- const deviceType = koffi.struct("CU_XIDeviceInfo", { deviceid: "int", name: "str", use: "int", attachment: "int", enabled: "int", num_classes: "int", classes: "void *" });
- const maskType = koffi.struct("CU_XIEventMask", { deviceid: "int", mask_len: "int", mask: "void *" });
- const open = x.func("void *XOpenDisplay(str name)"), close = x.func("int XCloseDisplay(void *display)");
- const display = open(null);
- if (!display) fail("desktop_unavailable", "Could not connect to the X11 desktop.");
- const root = x.func("ulong XDefaultRootWindow(void *display)")(display);
- const atom = (name) => x.func("ulong XInternAtom(void *display, str name, int only)")(display, name, 0);
- const property = x.func("int XGetWindowProperty(void *display, ulong window, ulong property, long offset, long length, int remove, ulong requested, _Out_ ulong *actual, _Out_ int *format, _Out_ ulong *count, _Out_ ulong *remaining, _Out_ void **data)");
- const free = x.func("int XFree(void *data)");
- const boundsOf = x.func("int XGetWindowAttributes(void *display, ulong window, _Out_ CU_XWindowAttributes *attributes)");
- const translate = x.func("int XTranslateCoordinates(void *display, ulong source, ulong destination, int x, int y, _Out_ int *root_x, _Out_ int *root_y, _Out_ ulong *child)");
- const queryTree = x.func("int XQueryTree(void *display, ulong window, _Out_ ulong *root, _Out_ ulong *parent, _Out_ void **children, _Out_ uint *count)");
- const sync = x.func("int XSync(void *display, int discard)");
- const setError = x.func("void *XSetErrorHandler(void *callback)");
- const errorProto = koffi.proto("int CU_XErrorHandler(void *display, void *event)");
- let xError = false;
- const errorCallback = koffi.register(() => { xError = true; return 0; }, koffi.pointer(errorProto));
- setError(errorCallback);
- const pending = x.func("int XPending(void *display)"), nextEvent = x.func("int XNextEvent(void *display, void *event)");
- const eventData = x.func("int XGetEventData(void *display, _Inout_ CU_XGenericEventCookie *cookie)");
- const freeEventData = x.func("void XFreeEventData(void *display, CU_XGenericEventCookie *cookie)");
- const queryDevices = xi.func("CU_XIDeviceInfo *XIQueryDevice(void *display, int deviceid, _Out_ int *count)");
- const freeDevices = xi.func("void XIFreeDeviceInfo(CU_XIDeviceInfo *devices)");
- const selectEvents = xi.func("int XISelectEvents(void *display, ulong window, CU_XIEventMask *masks, int count)");
- const major = [2], minor = [0];
- if (xi.func("int XIQueryVersion(void *display, _Inout_ int *major, _Inout_ int *minor)")(display, major, minor) !== 0) fail("input_unavailable", "XInput2 is required for interruption monitoring.");
- const testDevices = new Set();
- const refreshDevices = () => { const count = [0], ptr = queryDevices(display, 0, count); testDevices.clear(); if (ptr) { for (const d of koffi.decode(ptr, deviceType, count[0])) if (d.name.includes("XTEST")) testDevices.add(d.deviceid); freeDevices(ptr); } };
- refreshDevices();
- const eventMask = Buffer.alloc(4);
- for (const type of [11, 13, 14, 15, 16, 17]) eventMask[type >> 3] |= 1 << (type & 7);
- if (selectEvents(display, root, [{ deviceid: 0, mask_len: eventMask.length, mask: eventMask }], 1) !== 0) fail("input_unavailable", "X11 input monitoring could not start.");
- sync(display, 0);
- let injecting = false;
- const poll = () => {
- while (pending(display)) {
- const event = Buffer.alloc(192); nextEvent(display, event);
- const cookie = koffi.decode(event, cookieType);
- if (cookie.type !== 35 || !eventData(display, cookie)) continue;
- try {
- if (cookie.evtype === 11) refreshDevices();
- else if ([13, 14, 15, 16, 17].includes(cookie.evtype)) {
- const raw = koffi.decode(cookie.data, rawType);
- if (!injecting || !testDevices.has(raw.sourceid)) changed();
- }
- } finally { freeEventData(display, cookie); }
- }
- };
- function prop(id, name) {
- const actual = [0], format = [0], count = [0], remaining = [0], data = [null];
- if (property(display, id, atom(name), 0, 4096, 0, 0, actual, format, count, remaining, data) !== 0 || !data[0]) return [];
- try {
- if (format[0] === 32) return Array.from(koffi.decode(data[0], "ulong", Math.min(count[0], 4096)), Number);
- if (format[0] === 8) return Buffer.from(koffi.decode(data[0], "uint8_t", Math.min(count[0], 16384))).toString("utf8");
- return [];
- } finally { free(data[0]); }
- }
- const tree = (id) => {
- const parent = [0], parentRoot = [0], data = [null], count = [0];
- if (!queryTree(display, id, parentRoot, parent, data, count)) return { parent: 0, children: [] };
- try { return { parent: Number(parent[0]), children: data[0] ? Array.from(koffi.decode(data[0], "ulong", Math.min(count[0], 4096)), Number) : [] }; }
- finally { if (data[0]) free(data[0]); }
- };
- const owner = (id, target) => { for (let i = 0; id && i < 32; i++) { if (Number(id) === target) return true; id = tree(id).parent; } return false; };
- function window(id) {
- const box = {};
- if (!boundsOf(display, id, box) || box.map_state !== 2 || box.width <= 0 || box.height <= 0) fail("window_unavailable", "Keep the approved app visible on your desktop.");
- const pid = prop(id, "_NET_WM_PID")[0];
- let executable, identity;
- try {
- executable = readlinkSync("/proc/" + pid + "/exe");
- const stat = readFileSync("/proc/" + pid + "/stat", "utf8");
- identity = String(pid) + ":" + stat.slice(stat.lastIndexOf(")") + 2).split(" ")[19];
- } catch { fail("window_unavailable", "The app identity could not be verified."); }
- const xRoot = [0], yRoot = [0], child = [0];
- if (!translate(display, id, root, 0, 0, xRoot, yRoot, child)) fail("window_unavailable", "Could not read the window position.");
- const title = prop(id, "_NET_WM_NAME") || prop(id, "WM_NAME");
- return safeWindow({ id: Number(id), pid: Number(pid), executable, identity, appId: executable, appName: path.basename(executable), title: typeof title === "string" ? title.replaceAll("\0", "") : "App window", bounds: { x: xRoot[0], y: yRoot[0], width: box.width, height: box.height } });
- }
- const currentFocus = () => { const focused = [0], revert = [0]; x.func("int XGetInputFocus(void *display, _Out_ ulong *focus, _Out_ int *revert)")(display, focused, revert); return Number(focused[0]); };
- function guard(expected, expectedGeneration, position, deadline) {
- if (Date.now() >= deadline) fail("session_expired", "The approved session expired. Start a new session.");
- poll();
- if (generation !== expectedGeneration) fail("user_interacting", "Your input interrupted Computer Use. Choose Continue when ready.");
- const current = window(expected.id);
- if (current.identity !== expected.identity || JSON.stringify(current.bounds) !== JSON.stringify(expected.bounds)) fail("window_changed", "The approved window changed. Observe it again.");
- if (!owner(currentFocus(), expected.id)) fail("focus_changed", "The approved window must stay in front. Choose Continue when ready.");
- if (position) {
- const children = [], outputRoot = [0], child = [0], rx = [0], ry = [0], wx = [0], wy = [0], mask = [0];
- let parent = root;
- const queryPointer = x.func("int XQueryPointer(void *display, ulong window, _Out_ ulong *root, _Out_ ulong *child, _Out_ int *root_x, _Out_ int *root_y, _Out_ int *win_x, _Out_ int *win_y, _Out_ uint *mask)");
- for (let i = 0; i < 32; i++) { if (!queryPointer(display, parent, outputRoot, child, rx, ry, wx, wy, mask) || !child[0]) break; parent = Number(child[0]); children.push(parent); }
- if (!children.includes(expected.id)) fail("window_occluded", "Another window covers the pointer. Move it away and continue.");
- }
- return current;
- }
- // Capture the approved window's off-screen pixmap. XGetImage on the visible
- // window can include another app covering it on a non-compositing desktop.
- const composite = koffi.load("libXcomposite.so.1");
- const compositeMajor = [0], compositeMinor = [0];
- if (!composite.func("int XCompositeQueryVersion(void *display, _Out_ int *major, _Out_ int *minor)")(display, compositeMajor, compositeMinor) || (compositeMajor[0] === 0 && compositeMinor[0] < 2)) fail("capture_unavailable", "X11 Composite window capture is required.");
- const redirect = composite.func("void XCompositeRedirectWindow(void *display, ulong window, int update)");
- const unredirect = composite.func("void XCompositeUnredirectWindow(void *display, ulong window, int update)");
- const namePixmap = composite.func("ulong XCompositeNameWindowPixmap(void *display, ulong window)");
- const imageType = koffi.struct("CU_XImage", { width: "int", height: "int", xoffset: "int", format: "int", data: "void *", byte_order: "int", bitmap_unit: "int", bitmap_bit_order: "int", bitmap_pad: "int", depth: "int", bytes_per_line: "int", bits_per_pixel: "int", red_mask: "ulong", green_mask: "ulong", blue_mask: "ulong", obdata: "void *", functions: koffi.struct({ create_image: "void *", destroy_image: "void *", get_pixel: "void *", put_pixel: "void *", sub_image: "void *", add_pixel: "void *" }) });
- const visualType = koffi.struct("CU_XVisual", { ext_data: "void *", visualid: "ulong", class: "int", red_mask: "ulong", green_mask: "ulong", blue_mask: "ulong", bits_per_rgb: "int", map_entries: "int" });
- const getImage = x.func("CU_XImage *XGetImage(void *display, ulong drawable, int x, int y, uint width, uint height, ulong planes, int format)");
- const destroyImage = x.func("int XDestroyImage(CU_XImage *image)");
- const freePixmap = x.func("int XFreePixmap(void *display, ulong pixmap)");
- const redirected = new Set();
- const clearCapture = () => { for (const id of redirected) unredirect(display, id, 0); redirected.clear(); sync(display, 0); };
- const pngChunk = (type, data) => { const name = Buffer.from(type), length = Buffer.alloc(4), checksum = Buffer.alloc(4); length.writeUInt32BE(data.length); checksum.writeUInt32BE(crc32(data, crc32(name))); return Buffer.concat([length, name, data, checksum]); };
- async function capture(expected) {
- const current = window(expected.id);
- if (current.identity !== expected.identity || JSON.stringify(current.bounds) !== JSON.stringify(expected.bounds)) fail("window_changed", "The window changed. Observe again.");
- if (current.bounds.width * current.bounds.height > 16_777_216) fail("capture_unavailable", "This window is too large to capture. Resize it and observe again.");
- if (!redirected.has(current.id)) {
- xError = false; redirect(display, current.id, 0); sync(display, 0);
- if (xError) fail("capture_unavailable", "The approved window cannot be captured separately.");
- redirected.add(current.id);
- await new Promise((resolve) => setTimeout(resolve, 50));
- }
- xError = false;
- const pixmap = namePixmap(display, current.id); sync(display, 0);
- if (xError || !pixmap) fail("capture_unavailable", "The approved window cannot be captured separately.");
- let ptr;
- try {
- ptr = getImage(display, pixmap, 0, 0, current.bounds.width, current.bounds.height, ~0n, 2);
- if (!ptr) fail("capture_unavailable", "The window image is unavailable. Keep it visible and observe again.");
- const value = koffi.decode(ptr, imageType), bytes = value.bits_per_pixel / 8;
- const attributesValue = {};
- if (!boundsOf(display, current.id, attributesValue) || !attributesValue.visual) fail("capture_unavailable", "The window color format is unavailable.");
- const visual = koffi.decode(attributesValue.visual, visualType);
- if (![2, 3, 4].includes(bytes) || value.width !== current.bounds.width || value.height !== current.bounds.height || value.bytes_per_line < value.width * bytes || value.bytes_per_line * value.height > 67_108_864 || ![0, 1].includes(value.byte_order) || visual.class !== 4 || value.depth !== attributesValue.depth || !visual.red_mask || !visual.green_mask || !visual.blue_mask) fail("capture_unavailable", "This X11 image format is unavailable.");
- const source = Buffer.from(koffi.decode(value.data, "uint8_t", value.bytes_per_line * value.height));
- const ratio = Math.min(1, 1600 / Math.max(value.width, value.height));
- const width = Math.max(1, Math.floor(value.width * ratio)), height = Math.max(1, Math.floor(value.height * ratio));
- const rows = Buffer.alloc((width * 4 + 1) * height);
- const channels = [visual.red_mask, visual.green_mask, visual.blue_mask].map((mask) => ({ mask, shift: Math.log2((mask & -mask) >>> 0), max: mask >>> Math.log2((mask & -mask) >>> 0) }));
- for (let y = 0; y < height; y++) for (let column = 0; column < width; column++) {
- const offset = Math.floor(y / ratio) * value.bytes_per_line + Math.floor(column / ratio) * bytes;
- const pixel = value.byte_order === 0 ? source.readUIntLE(offset, bytes) : source.readUIntBE(offset, bytes);
- const destination = y * (width * 4 + 1) + 1 + column * 4;
- channels.forEach((channel, index) => { rows[destination + index] = Math.round(((pixel & channel.mask) >>> channel.shift) * 255 / channel.max); });
- rows[destination + 3] = 255;
- }
- const header = Buffer.alloc(13); header.writeUInt32BE(width); header.writeUInt32BE(height, 4); header[8] = 8; header[9] = 6;
- return { image: Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), pngChunk("IHDR", header), pngChunk("IDAT", deflateSync(rows, { level: 1 })), pngChunk("IEND", Buffer.alloc(0))]), size: { width, height } };
- } finally { if (ptr) destroyImage(ptr); freePixmap(display, pixmap); }
- }
- const rawKey = xt.func("int XTestFakeKeyEvent(void *display, uint key, int down, ulong delay)");
- const rawButton = xt.func("int XTestFakeButtonEvent(void *display, uint button, int down, ulong delay)");
- const rawMotion = xt.func("int XTestFakeMotionEvent(void *display, int screen, int x, int y, ulong delay)");
- const inject = (fn, args, check = true) => {
- const previous = generation; poll();
- if (check && previous !== generation) fail("user_interacting", "Your input interrupted Computer Use. Choose Continue when ready.");
- injecting = true;
- try { const result = fn(...args); sync(display, 0); poll(); return result; }
- finally { injecting = false; }
- };
- const fakeKey = (...args) => inject(rawKey, args);
- const fakeButton = (...args) => inject(rawButton, args);
- const fakeMotion = (...args) => inject(rawMotion, args);
- const keysym = x.func("ulong XStringToKeysym(str name)"), keycode = x.func("uint8_t XKeysymToKeycode(void *display, ulong symbol)");
- const downKeys = new Set();
- let pointerDown = false, wheelDown = 0;
- const release = () => { for (const key of downKeys) inject(rawKey, [display, key, 0, 0], false); downKeys.clear(); if (pointerDown) inject(rawButton, [display, 1, 0, 0], false); pointerDown = false; if (wheelDown) inject(rawButton, [display, wheelDown, 0, 0], false); wheelDown = 0; sync(display, 0); };
- const symbolAt = x.func("ulong XKeycodeToKeysym(void *display, uint8_t code, int index)");
- const key = (name, down) => { const symbol = typeof name === "number" ? name : keysym(name); const code = keycode(display, symbol); if (!code) fail("unsupported_action", "This character is unavailable in the active keyboard layout. Use an app connector for arbitrary Unicode text."); if (down) downKeys.add(code); else downKeys.delete(code); if (!fakeKey(display, code, down ? 1 : 0, 0)) fail("input_unavailable", "X11 rejected keyboard input."); sync(display, 0); };
- return {
- poll, window, release, check: guard, capture, clearCapture, available: () => true,
- list() { const ids = prop(root, "_NET_CLIENT_LIST_STACKING"); const result = []; for (const id of Array.isArray(ids) && ids.length ? ids : tree(root).children) { try { result.push(window(id)); } catch {} } return result; },
- focus(expected) {
- if (window(expected.id).identity !== expected.identity) fail("window_unavailable", "The approved app has closed.");
- x.func("int XRaiseWindow(void *display, ulong window)")(display, expected.id);
- x.func("int XSetInputFocus(void *display, ulong window, int revert, ulong time)")(display, expected.id, 2, 0);
- sync(display, 0); poll();
- },
- async act(expected, expectedGeneration, action, deadline) {
- const checked = (position) => guard(expected, expectedGeneration, position, deadline);
- try {
- checked();
- if (action.type === "type") {
- for (const char of action.text) {
- checked();
- const cp = char.codePointAt(0);
- const symbol = char === "\n" ? keysym("Return") : char === "\t" ? keysym("Tab") : cp <= 255 ? cp : 0x01000000 + cp;
- const code = keycode(display, symbol);
- if (!code || (Number(symbolAt(display, code, 0)) !== symbol && Number(symbolAt(display, code, 1)) !== symbol)) fail("unsupported_action", "This character is unavailable in the active keyboard layout. Use an app connector for arbitrary Unicode text.");
- const shifted = Number(symbolAt(display, code, 1)) === symbol && Number(symbolAt(display, code, 0)) !== symbol;
- if (shifted) key("Shift_L", true);
- key(symbol, true); key(symbol, false);
- if (shifted) key("Shift_L", false);
- await new Promise((resolve) => setTimeout(resolve, 2));
- }
- } else if (action.type === "key") {
- const modifiers = keys(action), name = { Enter: "Return", Space: "space", PageUp: "Prior", PageDown: "Next" }[action.key] ?? action.key;
- for (const modifier of modifiers) key({ ctrl: "Control_L", shift: "Shift_L", alt: "Alt_L" }[modifier], true);
- checked(); key(name, true); key(name, false);
- } else if (["click", "double_click", "scroll", "drag"].includes(action.type)) {
- const points = action.type === "drag" ? action.path : [action];
- for (const p of points) {
- checked(); fakeMotion(display, -1, expected.bounds.x + Math.round(p.x), expected.bounds.y + Math.round(p.y), 0); sync(display, 0); checked(p);
- if (action.type === "drag" && !pointerDown) { pointerDown = true; fakeButton(display, 1, 1, 0); sync(display, 0); }
- await new Promise((resolve) => setTimeout(resolve, action.type === "scroll" ? 32 : action.type === "drag" ? 12 : 2));
- }
- checked(points.at(-1));
- if (action.type === "scroll") {
- const button = action.axis === "horizontal" ? action.delta > 0 ? 7 : 6 : action.delta > 0 ? 5 : 4;
- // Let the app process pointer entry and distinct wheel ticks.
- // Keep releases tracked so person input or Stop cannot leave a button held.
- for (let i = 0; i < Math.abs(action.delta); i++) {
- checked(points.at(-1)); wheelDown = button; fakeButton(display, button, 1, 0); sync(display, 0);
- await new Promise((resolve) => setTimeout(resolve, 8));
- fakeButton(display, button, 0, 0); wheelDown = 0; sync(display, 0);
- await new Promise((resolve) => setTimeout(resolve, 12));
- }
- } else if (action.type !== "drag") {
- for (let i = 0; i < (action.type === "double_click" ? 2 : 1); i++) { checked(points.at(-1)); pointerDown = true; fakeButton(display, 1, 1, 0); fakeButton(display, 1, 0, 0); pointerDown = false; sync(display, 0); if (i === 0 && action.type === "double_click") await new Promise((resolve) => setTimeout(resolve, 60)); }
- }
- } else fail("unsupported_action", "Use click, double_click, drag, scroll, type, or key.");
- } finally { release(); }
- },
- close() { release(); clearCapture(); close(display); setError(null); koffi.unregister(errorCallback); },
- };
-}
-let timer;
-const startPolling = () => {
- if (timer) return;
- timer = setInterval(() => { try { backend?.poll(); } catch { changed(); } }, 10);
-};
-const stopPolling = () => {
- if (!timer) return;
- clearInterval(timer);
- timer = undefined;
-};
-try {
- backend = process.platform === "win32" ? windows() : x11();
- if (process.platform !== "win32") startPolling();
- parentPort.postMessage({ event: "ready", supported: true });
-} catch (error) {
- parentPort.postMessage({ event: "ready", supported: false, code: error.code ?? "native_unavailable", error: error.message });
-}
-let busy = false;
-parentPort.on("message", async ({ id, method, params }) => {
- let ownsBusy = false;
- try {
- if (method === "shutdown") {
- generation++; backend?.release(); backend?.stopMonitoring?.(); stopPolling();
- while (busy) await new Promise((resolve) => setTimeout(resolve, 10));
- backend?.close(); backend = null;
- parentPort.postMessage({ id, result: {} }); parentPort.close(); return;
- }
- if (!backend) fail("native_unavailable", "Desktop access is unavailable.");
- if (method === "start") {
- if (busy) fail("busy", "Desktop operations must be sequential.");
- backend.startMonitoring?.();
- if (process.platform === "win32") startPolling();
- parentPort.postMessage({ id, result: {} });
- return;
- }
- if (method === "stop") {
- generation++;
- backend.release();
- backend.stopMonitoring?.();
- if (process.platform === "win32") stopPolling();
- backend.clearCapture?.();
- parentPort.postMessage({ id, result: {} });
- return;
- }
- if (busy) fail("busy", "Desktop operations must be sequential.");
- busy = true; ownsBusy = true;
- backend.poll();
- let result;
- if (method === "list") result = backend.list();
- else if (method === "window") result = backend.window(params.id);
- else if (method === "focus") { backend.focus(params.window); result = { generation }; }
- else if (method === "state") result = { generation, available: backend.available() };
- else if (method === "capture") { result = await backend.capture(params.window); }
- else if (method === "check") { backend.check(params.window, params.generation, undefined, params.deadline); result = {}; }
- else if (method === "act") { await backend.act(params.window, params.generation, params.action, params.deadline); result = { generation }; }
- else fail("unknown_method", "Unknown native operation.");
- parentPort.postMessage({ id, result });
- } catch (error) {
- parentPort.postMessage({ id, error: { code: error.code ?? "native_error", message: error.message } });
- } finally { if (ownsBusy) busy = false; }
-});
-parentPort.on("close", () => { stopPolling(); backend?.close(); });
diff --git a/apps/desktop/electron/computer-use-native.mjs b/apps/desktop/electron/computer-use-native.mjs
deleted file mode 100644
index f6dbce0e..00000000
--- a/apps/desktop/electron/computer-use-native.mjs
+++ /dev/null
@@ -1,51 +0,0 @@
-import { spawn } from "node:child_process";
-import { createRequire } from "node:module";
-import { fileURLToPath } from "node:url";
-/** @param {unknown} value @returns {value is Record<string, unknown>} */
-function record(value) { return typeof value === "object" && value !== null && !Array.isArray(value); }
-export function createNativeDesktop() {
- const worker = spawn(process.execPath, [fileURLToPath(new URL("./computer-use-native-worker.mjs", import.meta.url))], { env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" }, stdio: ["ignore", "ignore", "ignore", "ipc"], windowsHide: true, serialization: "advanced" });
- const pending = new Map();
- const listeners = new Set();
- let nextId = 0, closed = false;
- let readyResolve;
- const ready = new Promise((resolve) => { readyResolve = resolve; });
- const readyTimer = setTimeout(() => readyResolve({ supported: false, error: "Desktop access did not start. Restart OmniRush.ai." }), 10_000);
- worker.on("message", (value) => {
- if (!record(value)) return;
- if (value.event === "ready") { clearTimeout(readyTimer); readyResolve(value); }
- if (value.event === "person-input") for (const listener of listeners) listener(value);
- const request = pending.get(value.id);
- if (!request) return;
- pending.delete(value.id); clearTimeout(request.timer);
- if (record(value.error) && typeof value.error.message === "string") request.reject(Object.assign(new Error(value.error.message), { code: value.error.code }));
- else request.resolve(value.result);
- });
- const fail = (error) => {
- clearTimeout(readyTimer); readyResolve({ supported: false, error: error.message });
- closed = true;
- for (const request of pending.values()) { clearTimeout(request.timer); request.reject(error); }
- pending.clear();
- for (const listener of listeners) listener({ event: "unavailable" });
- };
- worker.on("error", fail);
- worker.on("exit", () => fail(new Error("Desktop access stopped. Restart OmniRush.ai.")));
- return {
- ready,
- grantForeground() {
- if (process.platform !== "win32" || !worker.pid) return;
- const koffi = createRequire(import.meta.url)("koffi");
- koffi.load("user32.dll").func("int __stdcall AllowSetForegroundWindow(uint32_t processId)")(worker.pid);
- },
- onInput(listener) { listeners.add(listener); return () => listeners.delete(listener); },
- call(method, params = {}) {
- if (closed) return Promise.reject(new Error("Desktop access has stopped."));
- const id = ++nextId;
- return new Promise((resolve, reject) => {
- const timer = setTimeout(() => { pending.delete(id); reject(new Error("Desktop operation timed out.")); }, 10_000);
- pending.set(id, { resolve, reject, timer }); worker.send({ id, method, params });
- });
- },
- async close() { if (closed) return; await this.call("shutdown").catch(() => {}); closed = true; if (worker.exitCode === null) worker.kill(); },
- };
-}
diff --git a/apps/desktop/electron/computer-use-portable-host.mjs b/apps/desktop/electron/computer-use-portable-host.mjs
deleted file mode 100644
index be364c06..00000000
--- a/apps/desktop/electron/computer-use-portable-host.mjs
+++ /dev/null
@@ -1,250 +0,0 @@
-// Windows / X11 use the same MCP contract and main-window-only approvals as Mac.
-// The private native bridge is never reachable over the MCP transport.
-import { createServer } from "node:net";
-import { randomBytes, randomUUID, createHash, timingSafeEqual } from "node:crypto";
-import { mkdir, writeFile, rename, unlink } from "node:fs/promises";
-import path from "node:path";
-import { createNativeDesktop } from "./computer-use-native.mjs";
-
-const modes = ["observe", "control"];
-const methods = new Set(["initialize", "ping", "tools/list", "tools/call", "notifications/initialized", "notifications/cancelled"]);
-const tool = (name, description, properties, required = []) => ({ name, description, inputSchema: { type: "object", properties, required, additionalProperties: false } });
-const string = { type: "string" };
-const session = { session_id: string };
-const tools = [
- tool("computer_discover", "List running app identities without window contents. Start an app yourself if it is closed. Windows and Linux X11 support observe and foreground control; assist is unavailable.", {}),
- tool("computer_open_session", "Ask the person to approve one visible app window. Never infer consent. Wait for this tool to return before observing.", { app_id: string, pid: { type: "integer" }, mode: { type: "string", enum: modes }, purpose: string }, ["app_id", "purpose"]),
- tool("computer_observe", "Get a fresh screenshot of the approved window. Visual models are required. Coordinates for actions are screenshot pixels, relative to the window. Observe again after every action or failure.", { ...session, include_image: { type: "boolean" } }, ["session_id"]),
- tool("computer_act", "Perform one action using a fresh, single-use observation. Supported: click/double_click (x,y), drag (path [{x,y}]), scroll (x,y,delta in steps (positive down/right),axis), type (text), key (key,modifiers ctrl/shift/alt). Foreground only. Never retry an action when may_have_acted is true; observe and inspect its result.", { ...session, observation_id: string, request_id: string, action: { type: "object", properties: { type: { type: "string", enum: ["click", "double_click", "drag", "scroll", "type", "key"] }, x: { type: "number" }, y: { type: "number" }, path: { type: "array", items: { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false } }, delta: { type: "integer" }, axis: { type: "string", enum: ["vertical", "horizontal"] }, text: string, key: string, modifiers: { type: "array", items: { type: "string", enum: ["ctrl", "shift", "alt"] } } }, required: ["type"], additionalProperties: false } }, ["session_id", "observation_id", "request_id", "action"]),
- tool("computer_session_status", "Read the session status. After person input, wait for the person to choose Continue; the agent cannot resume itself.", session, ["session_id"]),
- tool("computer_close_session", "End access and release mouse and keyboard input.", session, ["session_id"]),
-];
-const failure = (code, message, extra = {}) => ({ ok: false, code, message, ...extra });
-const content = (state, image) => ({ content: [{ type: "text", text: JSON.stringify(state) }, ...(image ? [{ type: "image", mimeType: "image/png", data: image.toString("base64") }] : [])], isError: state.ok !== true });
-const digest = (image) => createHash("sha256").update(image).digest("hex");
-const sameBounds = (a, b) => JSON.stringify(a) === JSON.stringify(b);
-function validateAction(action, size) {
- if (!action || typeof action !== "object" || Array.isArray(action)) throw new Error("An action object is required.");
- const point = (p) => { if (!p || !Number.isFinite(p.x) || !Number.isFinite(p.y) || p.x < 0 || p.y < 0 || p.x >= size.width || p.y >= size.height) throw new Error("The point must be inside the approved screenshot."); };
- if (["click", "double_click", "scroll"].includes(action.type)) point(action);
- else if (action.type === "drag") { if (!Array.isArray(action.path) || action.path.length < 2 || action.path.length > 64) throw new Error("A drag needs 2 to 64 points."); action.path.forEach(point); }
- else if (action.type === "type") { if (typeof action.text !== "string" || action.text.length < 1 || action.text.length > 1000 || /[\u0000-\u0008\u000b\u000c\u000e-\u001f]/.test(action.text)) throw new Error("Text must contain 1 to 1000 printable characters."); }
- else if (action.type === "key") { if (typeof action.key !== "string" || action.key.length > 20) throw new Error("A supported key name is required."); }
- else throw new Error("This action is unavailable. Use click, double_click, drag, scroll, type, or key.");
- if (action.type === "scroll" && (!Number.isInteger(action.delta) || Math.abs(action.delta) > 20 || action.delta === 0 || !["vertical", "horizontal", undefined].includes(action.axis))) throw new Error("Scroll needs 1 to 20 steps and a vertical or horizontal axis.");
-}
-export async function createPortableComputerUseHost({ profile, capture, preview }) {
- const native = createNativeDesktop();
- const availability = await native.ready;
- const captureWindow = process.platform === "linux" ? (window) => native.call("capture", { window }) : capture;
- const directory = path.join(profile, "computer-use");
- await mkdir(directory, { recursive: true, mode: 0o700 });
- const discoveryPath = path.join(directory, "connection.json");
- const token = randomBytes(32).toString("hex");
- const connections = new Map();
- let lease = null, closed = false;
- const state = () => [...connections.values()].flatMap((entry) => entry.session ? [{
- connectionId: entry.id, id: entry.session.id, phase: entry.session.phase, appName: entry.session.window.appName,
- task: entry.session.purpose, mode: entry.session.mode, windows: entry.session.windows.map((w) => ({ id: w.id, title: w.title })),
- windowTitle: entry.session.window.title, windowBounds: entry.session.window.bounds, status: entry.session.status, recoverable: false,
- canContinue: entry.session.phase === "paused", previewVisible: entry.session.previewVisible,
- remainingSeconds: Math.max(0, Math.ceil((entry.session.expiresAt - Date.now()) / 1000)),
- }] : []);
- const update = () => preview?.update(state());
- async function stop(entry) {
- const value = entry.session;
- if (!value) return;
- entry.session = null; value.observation = null; value.approval?.resolve(failure("approval_cancelled", "The request ended without access."));
- if (lease === entry.id) lease = null;
- await native.call("stop").catch(() => {});
- update();
- }
- function pause(entry, message) {
- const value = entry.session;
- if (!value || value.phase === "approval") return;
- value.phase = "paused"; value.observation = null; value.status = message;
- void native.call("stop").catch(() => {}); update();
- }
- native.onInput(() => {
- for (const entry of connections.values()) if (entry.session?.phase === "working") pause(entry, "Your input interrupted control. Choose Continue when you are ready.");
- });
- const server = createServer((socket) => {
- if (closed || connections.size >= 16) { socket.destroy(); return; }
- const entry = { id: randomUUID(), session: null, socket, busy: false };
- connections.set(entry.id, entry);
- let incoming = "", authenticated = false;
- const authTimer = setTimeout(() => socket.destroy(), 5000);
- socket.setEncoding("utf8");
- const close = () => { clearTimeout(authTimer); connections.delete(entry.id); void stop(entry); };
- socket.once("close", close); socket.on("error", () => socket.destroy());
- const send = (message) => { if (!socket.destroyed) socket.write(JSON.stringify(message) + "\n"); };
- const respond = async (message) => {
- if (!message || message.jsonrpc !== "2.0" || !methods.has(message.method)) {
- send({ jsonrpc: "2.0", id: message?.id ?? null, error: { code: -32601, message: "Method not available to the agent." } }); return;
- }
- if (message.method === "notifications/cancelled") { if (entry.session?.approvalRequest === message.params?.requestId) await stop(entry); return; }
- if (message.method === "notifications/initialized") return;
- let result;
- try {
- if (message.method === "initialize") result = { protocolVersion: "2025-11-25", capabilities: { tools: {} }, serverInfo: { name: "omnirush-computer-use", version: "0.1.0" }, instructions: "Use only the approved app window. Foreground control pauses on person input. Only the person can approve or Continue. Observe after every action; never retry partial input automatically. For Blender background work prefer its Python CLI or an app connector." };
- else if (message.method === "ping") result = {};
- else if (message.method === "tools/list") result = { tools };
- else {
- const name = message.params?.name, args = message.params?.arguments ?? {};
- if (entry.busy && !["computer_session_status", "computer_close_session"].includes(name)) result = content(failure("busy", "Wait for the previous operation."));
- else {
- const ownsBusy = !["computer_session_status", "computer_close_session"].includes(name);
- if (ownsBusy) entry.busy = true;
- try { result = await call(entry, name, args, message.id); }
- finally { if (ownsBusy) entry.busy = false; }
- }
- }
- if (message.id !== undefined) send({ jsonrpc: "2.0", id: message.id, result });
- } catch (error) { if (message.id !== undefined) send({ jsonrpc: "2.0", id: message.id, result: content(failure(error.code ?? "desktop_error", error.message)) }); }
- };
- socket.on("data", (chunk) => {
- incoming += chunk;
- if (incoming.length > 1_048_576) { socket.destroy(); return; }
- let end;
- while ((end = incoming.indexOf("\n")) >= 0) {
- const line = incoming.slice(0, end); incoming = incoming.slice(end + 1);
- let message; try { message = JSON.parse(line); } catch { socket.destroy(); return; }
- if (!message || typeof message !== "object" || Array.isArray(message)) { socket.destroy(); return; }
- if (!authenticated) {
- const candidate = typeof message.token === "string" ? Buffer.from(message.token) : Buffer.alloc(0);
- if (candidate.length !== token.length || !timingSafeEqual(candidate, Buffer.from(token))) { socket.destroy(); return; }
- authenticated = true; clearTimeout(authTimer); continue;
- }
- void respond(message);
- }
- });
- });
- async function active(entry, id) {
- const value = entry.session;
- if (!value || value.id !== id || value.phase === "approval") return null;
- if (Date.now() >= value.expiresAt) { await stop(entry); return null; }
- if (Date.now() - value.lastOperation > 120_000 && value.phase === "working") pause(entry, "Paused after two minutes without work. Choose Continue when ready.");
- return value;
- }
- async function observation(entry, value, includeImage) {
- if (value.phase !== "working") return content(failure("session_paused", value.status, { next: "human_takeover" }));
- const before = await native.call("state");
- const window = await native.call("window", { id: value.window.id });
- if (window.identity !== value.window.identity) { await stop(entry); return content(failure("window_unavailable", "The approved app closed or restarted. Start a new session.")); }
- let captured = await captureWindow(window);
- let stable = false;
- for (let attempt = 0; attempt < 4; attempt++) {
- await new Promise((resolve) => setTimeout(resolve, 50));
- const next = await captureWindow(window);
- if (digest(next.image) === digest(captured.image)) { captured = next; stable = true; break; }
- captured = next;
- }
- if (!stable) return content(failure("window_changing", "Wait for the app to finish updating, then observe again."));
-
- const after = await native.call("state");
- if (entry.session !== value || value.phase !== "working" || before.generation !== after.generation || !after.available) { pause(entry, "Desktop input or availability changed. Choose Continue when ready."); return content(failure("session_paused", "Observe after the person continues.")); }
- const id = randomUUID();
- value.window = window; value.lastOperation = Date.now();
- value.observation = { id, time: Date.now(), generation: after.generation, window, size: captured.size, hash: digest(captured.image) };
- preview?.image(captured.image);
- update();
- return content({ ok: true, session_id: value.id, observation_id: id, window_title: window.title, image_size: captured.size, window_bounds: window.bounds, coordinates: "screenshot pixels relative to the approved window", elements: [], modes, requires_visual_model: true }, includeImage ? captured.image : undefined);
- }
- async function call(entry, name, args, requestId) {
- if (!tools.some((item) => item.name === name)) return content(failure("unknown_tool", "Unknown Computer Use tool."));
- if (!availability.supported) return content(failure(availability.code ?? "desktop_unavailable", availability.error));
- if (name === "computer_discover") {
- const windows = await native.call("list");
- const apps = new Map();
- for (const window of windows) apps.set(window.pid, { app_id: window.appId, pid: window.pid, name: window.appName });
- return content({ ok: true, modes, background_control: false, apps: [...apps.values()] });
- }
- if (name === "computer_open_session") {
- if (!modes.includes(args.mode ?? "observe")) return content(failure("unsupported_mode", "This platform supports read-only observation or foreground mouse and keyboard control."));
- if (typeof args.app_id !== "string" || typeof args.purpose !== "string" || !args.purpose.trim() || args.purpose.length > 500) return content(failure("invalid_request", "Name a running app and describe the task in up to 500 characters."));
- if (lease) return content(failure("session_busy", "Another Computer Use request is active. End it first."));
- lease = entry.id;
- try {
- const windows = (await native.call("list")).filter((window) => window.appId === args.app_id && (args.pid === undefined || window.pid === args.pid));
- if (!windows.length) { lease = null; return content(failure("app_unavailable", "Open the app yourself, then discover its windows again.")); }
- const value = { id: randomUUID(), window: windows[0], windows, mode: args.mode ?? "observe", purpose: args.purpose, phase: "approval", previewVisible: true, expiresAt: Date.now() + 900_000, lastOperation: Date.now(), attempts: 0, requests: new Set(), observation: null, approvalRequest: requestId };
- entry.session = value;
- const approval = new Promise((resolve) => { value.approval = { resolve }; });
- const timeout = setTimeout(() => { if (entry.session === value) void stop(entry); }, 60_000);
- update();
- try {
- const result = await approval;
- if (result.ok !== true) return content(result);
- return content({ ok: true, session_id: value.id, state: "active", app_id: value.window.appId, pid: value.window.pid, mode: value.mode, window_title: value.window.title, expires_in_seconds: 900, background_control: false });
- } finally { clearTimeout(timeout); value.approval = null; }
- } catch (error) { if (lease === entry.id) lease = null; throw error; }
- }
- const value = await active(entry, args.session_id);
- if (!value) return content(failure("session_unavailable", "This session has ended or belongs to another client."));
- if (name === "computer_close_session") { await stop(entry); return content({ ok: true }); }
- if (name === "computer_session_status") return content({ ok: true, session_id: value.id, state: value.phase === "working" ? "active" : "paused", phase: value.phase, mode: value.mode, background_control: false, panel_visible: value.previewVisible, remaining_seconds: Math.max(0, Math.ceil((value.expiresAt - Date.now()) / 1000)), message: value.status });
- if (name === "computer_observe") return observation(entry, value, args.include_image !== false);
- if (value.mode !== "control") return content(failure("scope_denied", "This session is read-only."));
- if (value.phase !== "working") return content(failure("session_paused", value.status, { next: "human_takeover" }));
- if (++value.attempts > 200) { await stop(entry); return content(failure("action_limit", "Start a new session after 200 action attempts.")); }
- const observed = value.observation; value.observation = null;
- if (typeof args.request_id !== "string" || !args.request_id || args.request_id.length > 100 || value.requests.has(args.request_id)) return content(failure("duplicate_request", "Use a unique request ID. Observe the current state instead of repeating input."));
- value.requests.add(args.request_id);
- if (!observed || observed.id !== args.observation_id || Date.now() - observed.time > 15_000) return content(failure("stale_observation", "Observe the window again before acting.", { may_have_acted: false }));
- try { validateAction(args.action, observed.size); } catch (error) { return content(failure("invalid_action", error.message, { may_have_acted: false })); }
- const current = await native.call("window", { id: value.window.id });
- if (current.identity !== observed.window.identity || !sameBounds(current.bounds, observed.window.bounds)) return content(failure("window_changed", "The window moved or changed. Observe again.", { may_have_acted: false }));
- try { await native.call("check", { window: current, generation: observed.generation, deadline: value.expiresAt }); }
- catch (error) { pause(entry, error.message); return content(failure(error.code ?? "input_interrupted", error.message, { may_have_acted: false, next: "human_takeover" })); }
- const fresh = await captureWindow(current);
- if (digest(fresh.image) !== observed.hash) return content(failure("stale_observation", "The window changed. Observe again.", { may_have_acted: false }));
- if (entry.session !== value || value.phase !== "working") return content(failure("session_paused", "The session was interrupted.", { may_have_acted: false }));
- const scale = (p) => ({ ...p, x: Math.min(current.bounds.width - 1, Math.floor(p.x * current.bounds.width / observed.size.width)), y: Math.min(current.bounds.height - 1, Math.floor(p.y * current.bounds.height / observed.size.height)) });
- const action = args.action.type === "drag" ? { ...args.action, path: args.action.path.map(scale) } : ["click", "double_click", "scroll"].includes(args.action.type) ? scale(args.action) : args.action;
- try {
- await native.call("act", { window: current, generation: observed.generation, action, deadline: value.expiresAt });
- value.lastOperation = Date.now();
- return content({ ok: true, session_id: value.id, request_id: args.request_id, dispatched: true, next: "observe" });
- } catch (error) {
- pause(entry, error.message);
- return content(failure(error.code ?? "input_interrupted", error.message, { may_have_acted: true, next: "human_takeover" }));
- }
- }
- await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => resolve(undefined)); });
- const address = server.address();
- if (!address || typeof address === "string") throw new Error("Computer Use could not create its local connection.");
- const endpoint = { version: 1, host: "127.0.0.1", port: address.port, token };
- const temp = discoveryPath + "." + randomUUID();
- await writeFile(temp, JSON.stringify(endpoint), { mode: 0o600, flag: "wx" });
- await rename(temp, discoveryPath);
- const expiry = setInterval(() => { for (const entry of connections.values()) { const value = entry.session; if (!value) continue; if (Date.now() >= value.expiresAt) void stop(entry); else if (value.phase === "working" && Date.now() - value.lastOperation > 120_000) pause(entry, "Paused after two minutes without work. Choose Continue when ready."); } update(); }, 1000);
- const host = {
- discoveryPath, state, availability,
- pause(connectionId) { const entry = connections.get(connectionId); if (entry) pause(entry, "You took over. Choose Continue in OmniRush.ai when ready."); },
- async apps() { if (!availability.supported) return { ok: false, apps: [] }; const windows = await native.call("list"); return { ok: true, apps: [...new Set(windows.map((w) => w.appName))] }; },
- async action(input) {
- const entry = connections.get(input?.connectionId), value = entry?.session;
- if (!value || value.id !== input.id || !["approve", "deny", "resume", "stop", "hide", "show"].includes(input.action)) throw new Error("This Computer Use request has ended.");
- if (input.action === "stop" || input.action === "deny") return stop(entry);
- if (input.action === "hide") { value.previewVisible = false; update(); return; }
- if (input.action === "show") { value.previewVisible = true; update(); return; }
- if ((input.action === "approve" && value.phase !== "approval") || (input.action === "resume" && value.phase !== "paused")) throw new Error("This action is unavailable.");
- const selected = input.action === "approve" ? value.windows.find((w) => w.id === input.windowId) : value.window;
- if (!selected) throw new Error("Choose a window.");
- const current = await native.call("window", { id: selected.id });
- if (current.identity !== selected.identity) { await stop(entry); throw new Error("The app restarted. Start a new session."); }
- if (value.mode === "control") {
- native.grantForeground?.();
- await native.call("focus", { window: current });
- }
- await native.call("start");
- if (entry.session !== value) throw new Error("The request ended.");
- value.window = current; value.phase = "working"; value.status = ""; value.lastOperation = Date.now(); value.observation = null;
- if (input.action === "approve") value.expiresAt = Date.now() + 900_000;
- value.approval?.resolve({ ok: true }); update();
- },
- async close() { if (closed) return; closed = true; clearInterval(expiry); for (const entry of connections.values()) { await stop(entry); entry.socket.destroy(); } server.close(); preview?.close(); await native.close(); await unlink(discoveryPath).catch(() => {}); },
- };
- preview?.bind(host);
- return host;
-}
diff --git a/apps/desktop/electron/computer-use-preview.mjs b/apps/desktop/electron/computer-use-preview.mjs
index 293fdfd2..995f4798 100644
--- a/apps/desktop/electron/computer-use-preview.mjs
+++ b/apps/desktop/electron/computer-use-preview.mjs
@@ -1,25 +1,15 @@
-import { BrowserWindow, ipcMain, desktopCapturer, screen } from "electron";
+import { BrowserWindow, ipcMain, screen } from "electron";
import { fileURLToPath } from "node:url";
-export async function captureComputerWindow(window) {
- const limit = 1600, ratio = Math.min(1, limit / Math.max(window.bounds.width, window.bounds.height));
- const sources = await desktopCapturer.getSources({ types: ["window"], thumbnailSize: { width: Math.round(window.bounds.width * ratio), height: Math.round(window.bounds.height * ratio) }, fetchWindowIcons: false });
- const source = sources.find((item) => Number(item.id.split(":")[1]) === window.id);
- if (!source || source.thumbnail.isEmpty()) throw Object.assign(new Error("This window could not be captured. Keep it visible; protected video and some GPU windows may not support capture."), { code: "capture_unavailable" });
- const size = source.thumbnail.getSize();
- if (size.width <= 0 || size.height <= 0 || Math.abs(size.width / size.height - window.bounds.width / window.bounds.height) > 0.04) throw Object.assign(new Error("The captured window size changed. Observe again."), { code: "window_changed" });
- return { image: source.thumbnail.toPNG(), size };
-}
export function createComputerPreview() {
let host, panel, current, placedSession;
const channel = "omnirush-computer-use-preview-action";
const action = (_event, value) => {
if (!panel || _event.sender !== panel.webContents || _event.senderFrame !== panel.webContents.mainFrame || !["stop", "hide", "pause"].includes(value) || !current) return;
- if (value === "pause") host.pause(current.connectionId);
- else void host.action({ connectionId: current.connectionId, id: current.id, action: value }).catch(() => {});
+ void host.control({ session_id: current.id, action: value === "pause" ? "takeover" : value }).catch(() => {});
};
ipcMain.on(channel, action);
const placement = () => {
- const nativeBounds = current.windowBounds;
+ const nativeBounds = current.windowBounds ?? { x: 0, y: 0, width: 800, height: 600 };
const bounds = process.platform === "win32" ? screen.screenToDipRect(null, nativeBounds) : nativeBounds;
const display = screen.getDisplayMatching(bounds), area = display.workArea;
const box = process.platform === "linux" && display.scaleFactor !== 1
@@ -44,18 +34,20 @@ export function createComputerPreview() {
placedSession = current.id;
panel.webContents.setWindowOpenHandler(() => ({ action: "deny" }));
panel.webContents.on("will-navigate", (event) => event.preventDefault());
- panel.on("close", (event) => { if (!current) return; event.preventDefault(); void host.action({ connectionId: current.connectionId, id: current.id, action: "hide" }).catch(() => {}); });
+ panel.on("close", (event) => { if (!current) return; event.preventDefault(); void host.control({ session_id: current.id, action: "hide" }).catch(() => {}); });
panel.webContents.once("did-finish-load", () => { if (current?.previewVisible) { panel.webContents.send("omnirush-computer-use-preview-state", current); panel.showInactive(); } });
void panel.loadFile(fileURLToPath(new URL("./computer-use-preview.html", import.meta.url)));
};
return {
bind(value) { host = value; },
+ /** Host states (portable-host.mjs): the first working or paused session shows the pill. */
update(states) {
- current = states.find((value) => value.phase !== "approval");
+ const value = states.find((s) => s.phase === "working" || s.phase === "paused");
+ current = value ? { id: value.id, phase: value.phase, mode: value.mode, status: value.status, appName: value.app.app_name, remainingSeconds: value.seconds_left, previewVisible: !value.hidden, windowBounds: value.window_bounds ?? null } : null;
if (!current || !current.previewVisible) { panel?.hide(); return; }
show(); panel.webContents.send("omnirush-computer-use-preview-state", current);
},
- image(image) { panel?.webContents.send("omnirush-computer-use-preview-image", "data:image/png;base64," + image.toString("base64")); },
+ image(dataUrl) { panel?.webContents.send("omnirush-computer-use-preview-image", dataUrl); },
close() { current = null; panel?.destroy(); panel = null; ipcMain.removeListener(channel, action); },
};
}
diff --git a/apps/desktop/electron/computer-use.mjs b/apps/desktop/electron/computer-use.mjs
index 6c7cda78..06659060 100644
--- a/apps/desktop/electron/computer-use.mjs
+++ b/apps/desktop/electron/computer-use.mjs
@@ -1,42 +1,266 @@
-// Computer-use helper integration: locating the bundled ComputerUse.app,
-// permission checks (spawn --check for a fresh TCC read), running-app
-// listing for @App mentions, and opening the permission-setup GUI.
-// Extracted from main.mjs; consumed only by the desktop IPC registry.
+// Computer use in the desktop app (contract v2): the host runs in this main
+// process, behind the server switch (the local server's /computer-use/state:
+// GET /collect/capabilities `computer_use`, or OMNIRUSH_COMPUTER_USE=0/1).
+//
+// - Windows and Linux X11: the shared portable host + native worker
+// (apps/server/src/computer-use/shared/, the same files as the CLI's).
+// - macOS: the signed Swift helper (computer-use-host.mjs), same interface.
+//
+// The engine reaches the host only as the built-in MCP server `computer-use`
+// (the local server adds it from host.json; the id is reserved). Consent is
+// never the engine's: every open-session ask, and the per-action asks of the
+// chat's permission mode, are answered in the main window (desktop IPC from
+// its main frame, input checked for synthetic events), never by the agent.
import { spawn } from "node:child_process";
import { existsSync } from "node:fs";
+import { createRequire } from "node:module";
import path from "node:path";
-import { fileURLToPath } from "node:url";
+import { fileURLToPath, pathToFileURL } from "node:url";
import { app } from "electron";
import { createComputerUseHost } from "./computer-use-host.mjs";
-import { createPortableComputerUseHost } from "./computer-use-portable-host.mjs";
-import { captureComputerWindow, createComputerPreview } from "./computer-use-preview.mjs";
-
-// Computer Use is paused for now: no host starts, so an already configured
-// Computer Use connection fails to start instead of controlling the desktop.
-const COMPUTER_USE_PAUSED = true;
-const COMPUTER_USE_PAUSED_MESSAGE = "Computer Use is paused for now.";
-
-let hostPromise;
-async function computerUseHost() {
- if (COMPUTER_USE_PAUSED) throw new Error(COMPUTER_USE_PAUSED_MESSAGE);
- if (!hostPromise) {
- if (process.platform === "win32" || process.platform === "linux") {
- hostPromise = createPortableComputerUseHost({ profile: app.getPath("userData"), capture: captureComputerWindow, preview: createComputerPreview() }).catch((error) => { hostPromise = null; throw error; });
- return hostPromise;
- }
- const executable = resolveComputerUseExecutable();
- if (!executable) throw new Error("Computer Use helper unavailable.");
- hostPromise = createComputerUseHost({ profile: app.getPath("userData"), executable }).catch((error) => { hostPromise = null; throw error; });
+import { createComputerPreview } from "./computer-use-preview.mjs";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const require = createRequire(import.meta.url);
+const STATUS_CHANNEL = "omnirush:computer-use:status";
+const SYNTHETIC_MESSAGE = "Use your own mouse or keyboard to approve.";
+
+let deps = null;
+let server = null; // the last /computer-use/state answer
+let host = null;
+let hostPromise = null;
+let listener = null;
+let preview = null;
+let shared = null;
+let pollTimer = null;
+let refreshTimer = null;
+const pending = new Map(); // request_id → { request, resolve }
+
+function shippedModule(name) {
+ // Packaged: app.asar/server/dist; dev: apps/server/dist (OMNIRUSH_DEV_MODE prefers it).
+ const candidates = [
+ path.resolve(__dirname, "..", "server", "dist", "computer-use", "shared", name),
+ path.resolve(__dirname, "..", "..", "server", "dist", "computer-use", "shared", name),
+ path.resolve(__dirname, "..", "..", "server", "src", "computer-use", "shared", name),
+ ];
+ if (process.env.OMNIRUSH_DEV_MODE === "1") candidates.unshift(candidates.splice(1, 1)[0]);
+ const found = candidates.find((candidate) => existsSync(candidate));
+ if (!found) throw new Error("Computer use is missing from this build. Reinstall OmniRush.ai.");
+ return pathToFileURL(found).href;
+}
+
+async function sharedModules() {
+ if (!shared) {
+ const [hostModule, mediaModule, policyModule, contractModule, imageModule] = await Promise.all(["portable-host.mjs", "media.mjs", "policy.mjs", "contract.mjs", "image.mjs"].map((name) => import(shippedModule(name))));
+ shared = { hostModule, mediaModule, policyModule, contractModule, imageModule, ancestors: policyModule.readAncestors() };
+ }
+ return shared;
+}
+
+async function serverFetch(pathname, init = {}) {
+ const info = await deps?.serverInfo?.();
+ const token = info?.clientToken ?? info?.ownerToken;
+ if (!info?.baseUrl || !token) throw new Error("The local OmniRush.ai server is not ready.");
+ // loopback-fetch: the locally managed OmniRush.ai server.
+ const response = await fetch(`${info.baseUrl}${pathname}`, { ...init, headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", ...(init.headers ?? {}) }, signal: AbortSignal.timeout(10_000) });
+ if (!response.ok) throw new Error(`computer use route ${pathname}: ${response.status}`);
+ return response.json();
+}
+
+function supported() {
+ return process.platform === "win32" || process.platform === "linux" || process.platform === "darwin";
+}
+
+function statusNow() {
+ const sessions = host ? host.state() : [];
+ return {
+ enabled: server?.enabled === true,
+ supported: Boolean(host?.availability?.supported ?? supported()),
+ message: server?.enabled ? (host && !host.availability?.supported ? host.availability?.error ?? "Computer use is unavailable on this desktop." : null) : server?.setting?.message ?? (server ? "Computer use is turned off." : null),
+ mode: server?.mode ?? "normal",
+ sessions,
+ requests: [...pending.values()].map((entry) => entry.request),
+ };
+}
+
+function pushStatus() {
+ const window = deps?.getWindow?.();
+ if (window && !window.isDestroyed()) window.webContents.send(STATUS_CHANNEL, statusNow());
+ if (host) preview?.update(host.state());
+}
+
+async function iconOf(file) {
+ try {
+ if (!file || !existsSync(file)) return null;
+ const image = await app.getFileIcon(file, { size: "normal" });
+ return image.isEmpty() ? null : image.toDataURL();
+ } catch {
+ return null;
}
+}
+
+function ask(request) {
+ return new Promise((resolve) => {
+ pending.set(request.request_id, { request, resolve });
+ void host?.watchInput?.();
+ // Bring the main window up: the question is answered there.
+ const window = deps?.getWindow?.();
+ if (window && !window.isDestroyed() && !window.isFocused()) window.flashFrame?.(true);
+ pushStatus();
+ });
+}
+
+function settled(requestId) {
+ pending.delete(requestId);
+ pushStatus();
+}
+
+const approve = {
+ async open(request) {
+ const icon = await iconOf(request.app.bundle_or_exe);
+ const answer = await ask({ kind: "open", ...request, icon, chat: server?.chat ?? null });
+ settled(request.request_id);
+ return answer;
+ },
+ async act(request) {
+ const { policyModule } = await sharedModules();
+ const reasons = request.reasons.map((r) => ({ ...r, text: `${policyModule.ASK_REASONS[r.code] ?? r.code}${r.detail ? ` (${r.detail})` : ""}` }));
+ const answer = await ask({ kind: "act", ...request, reasons, icon: await iconOf(request.app.bundle_or_exe), chat: server?.chat ?? null });
+ settled(request.request_id);
+ return { allow: answer.decision === "allow" };
+ },
+};
+
+function hostSettings() {
+ const setting = server?.setting ?? {};
+ return { enabled: server?.enabled === true, real_screen: setting.real_screen !== false, limits: setting.limits ?? null, deny_apps: setting.deny_apps ?? [], mode: server?.mode ?? "normal", message: setting.message ?? null };
+}
+
+function hostContext() {
+ const metrics = (() => { try { return app.getAppMetrics().map((m) => m.pid); } catch { return []; } })();
+ return { ancestors: shared?.ancestors ?? [], self_pids: [process.pid, ...metrics], chat: server?.chat ?? null };
+}
+
+async function reportEvent(event) {
+ const session = host?.state().find((s) => s.id === event.session_id);
+ await serverFetch("/computer-use/event", { method: "POST", body: JSON.stringify({ event, ...(session?.chat ? { chat: session.chat } : {}) }) }).catch(() => {});
+}
+
+async function startHost() {
+ if (hostPromise) return hostPromise;
+ hostPromise = (async () => {
+ const { hostModule, mediaModule, policyModule, contractModule, imageModule } = await sharedModules();
+ const spool = server?.spool ? mediaModule.createMediaSpool(server.spool) : null;
+ const callbacks = {
+ spool, settings: hostSettings, context: hostContext, approve,
+ onEvent: (event) => { void reportEvent(event); },
+ onState: () => { pushStatus(); scheduleRefresh(); },
+ onFrame: (frame) => preview?.image(frame.data_url),
+ };
+ let created;
+ if (process.platform === "darwin") {
+ const executable = resolveComputerUseExecutable();
+ if (!executable) throw new Error("The Computer Use helper is unavailable. Rebuild or reinstall OmniRush.ai.");
+ created = await createComputerUseHost({ executable, ...callbacks, shared: { portable: hostModule, policy: policyModule, contract: contractModule, image: imageModule } });
+ } else {
+ const native = hostModule.createNativeWorker({ nodePath: process.execPath, nodeEnv: { ELECTRON_RUN_AS_NODE: "1" }, koffiPath: require.resolve("koffi") });
+ created = await hostModule.createPortableHost({ native, ...callbacks });
+ preview = createComputerPreview();
+ preview.bind(created);
+ }
+ host = created;
+ listener = await created.listen(path.join(app.getPath("userData"), "computer-use"));
+ const relay = app.isPackaged ? path.join(process.resourcesPath, "computer-use", "relay.mjs") : path.join(__dirname, "computer-use-relay.mjs");
+ const command = [process.execPath, relay, listener.discoveryPath];
+ const environment = { ELECTRON_RUN_AS_NODE: "1" };
+ await serverFetch("/computer-use/host", { method: "POST", body: JSON.stringify({ command, environment }) });
+ pushStatus();
+ return created;
+ })().catch((error) => { hostPromise = null; host = null; throw error; });
return hostPromise;
}
-async function getComputerUseState() { return hostPromise ? (await hostPromise).state() : []; }
-async function computerUseAction(value) { await (await computerUseHost()).action(value); }
-function getComputerUseMcpEnvironment() { return process.platform === "darwin" ? {} : { ELECTRON_RUN_AS_NODE: "1" }; }
-app.on("before-quit", () => { hostPromise?.then((host) => host.close()).catch(() => {}); });
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
+async function stopHost(by = "server") {
+ const current = host;
+ await serverFetch("/computer-use/host", { method: "POST", body: JSON.stringify({ host: null }) }).catch(() => {});
+ if (!current) return;
+ await current.stopAll(by).catch(() => {});
+ for (const entry of pending.values()) entry.resolve({ decision: "deny" });
+ pending.clear();
+ await listener?.close().catch(() => {});
+ await current.close().catch(() => {});
+ preview?.close(); preview = null;
+ host = null; hostPromise = null; listener = null;
+ pushStatus();
+}
+
+async function apply(next) {
+ server = next;
+ if (server?.enabled && supported()) await startHost().catch((error) => { deps?.log?.("computer use host did not start", error); });
+ else if (host) await stopHost("server");
+ pushStatus();
+}
+
+// While a session is open the switch is read again every 60 s: off closes it within a minute.
+function scheduleRefresh() {
+ const open = host?.state().some((s) => s.phase !== "ended") ?? false;
+ if (open && !refreshTimer) {
+ refreshTimer = setInterval(() => {
+ void serverFetch("/computer-use/refresh", { method: "POST", body: "{}" }).then(apply).catch(() => {});
+ }, 60_000);
+ refreshTimer.unref?.();
+ } else if (!open && refreshTimer) {
+ clearInterval(refreshTimer); refreshTimer = null;
+ }
+}
+
+/** Starts following the switch (main.mjs, once the local server runs). */
+export function initComputerUse(options) {
+ deps = options;
+ const poll = () => void serverFetch("/computer-use/state").then(apply).catch(() => {});
+ poll();
+ // The state route is local: the chat in use and the switch are read every few seconds.
+ pollTimer = setInterval(poll, 5_000);
+ pollTimer.unref?.();
+}
+
+app.on("before-quit", () => { if (pollTimer) clearInterval(pollTimer); void stopHost("agent"); });
+
+async function getComputerUseState() {
+ return statusNow();
+}
+
+/** An answer from the main window. Synthetic input (another program clicking Allow) is refused. */
+async function computerUseAnswer(answer) {
+ const entry = pending.get(answer?.request_id);
+ if (!entry) throw new Error("This request has ended.");
+ const allowing = answer.decision !== "deny";
+ if (allowing && host && await host.syntheticInput()) throw new Error(SYNTHETIC_MESSAGE);
+ const decision = entry.request.kind === "open" ? (["chat", "once", "deny"].includes(answer.decision) ? answer.decision : "deny") : (answer.decision === "allow" ? "allow" : "deny");
+ entry.resolve({ decision, ...(typeof answer.window_id === "string" ? { window_id: answer.window_id } : {}) });
+}
+
+async function computerUseControl(value) {
+ if (!host) throw new Error("Computer use is not running.");
+ if (value?.action === "continue" && await host.syntheticInput()) throw new Error(SYNTHETIC_MESSAGE);
+ await host.control({ session_id: value?.session_id, action: value?.action });
+ pushStatus();
+}
+
+function getComputerUseMcpEnvironment() { return { ELECTRON_RUN_AS_NODE: "1" }; }
+
+async function getComputerUseMcpCommand() {
+ if (!server?.enabled) throw new Error(server?.setting?.message ?? "Computer use is turned off.");
+ await startHost();
+ const relay = app.isPackaged ? path.join(process.resourcesPath, "computer-use", "relay.mjs") : path.join(__dirname, "computer-use-relay.mjs");
+ return [process.execPath, relay, listener.discoveryPath];
+}
+
+// ---------------------------------------------------------------------------
+// macOS helper: locating the bundled app, permission checks (spawn --check for
+// a fresh TCC read), running-app listing for @App mentions, permission setup.
+// ---------------------------------------------------------------------------
const COMPUTER_USE_HELPER_APP_NAME = "OmniRush.ai Computer Use.app";
const COMPUTER_USE_HELPER_EXECUTABLE = "ComputerUse";
@@ -48,156 +272,91 @@ function computerUseHelperAppPath() {
process.resourcesPath ? path.join(process.resourcesPath, "helpers", COMPUTER_USE_HELPER_APP_NAME) : null,
path.resolve(__dirname, "..", "resources", "helpers", COMPUTER_USE_HELPER_APP_NAME),
].filter(Boolean);
-
return candidates.find((candidate) => existsSync(candidate)) ?? null;
}
-async function getComputerUseMcpCommand() {
- if (COMPUTER_USE_PAUSED) throw new Error(COMPUTER_USE_PAUSED_MESSAGE);
- if (process.platform !== "darwin") {
- const host = await computerUseHost();
- if (!host.availability.supported) throw new Error(host.availability.error);
- const relay = app.isPackaged ? path.join(process.resourcesPath, "computer-use", "relay.mjs") : path.join(__dirname, "computer-use-relay.mjs");
- if (!existsSync(relay)) throw new Error("Computer Use is missing from this build. Reinstall OmniRush.ai.");
- return [process.execPath, relay, host.discoveryPath];
- }
- const helperExecutable = resolveComputerUseExecutable();
- if (helperExecutable) return [helperExecutable, "relay", (await computerUseHost()).socketPath];
-
- if (app.isPackaged) {
- throw new Error("OmniRush.ai Computer Use is missing from this OmniRush.ai build.");
- }
-
- throw new Error("The Computer Use helper is unavailable. Rebuild or reinstall OmniRush.ai.");
-}
-
-// ---------------------------------------------------------------------------
-// Permission checks — spawn the binary with --check, read stdout, done.
-// Check in the same launch context as setup. TCC can attribute a child process
-// differently from a helper opened separately through LaunchServices.
-// ---------------------------------------------------------------------------
-
function resolveComputerUseExecutable() {
- // 1. Explicit env override.
const explicit = process.env.OMNIRUSH_COMPUTER_USE_BINARY?.trim();
if (explicit && existsSync(explicit)) return explicit;
-
- // 2. .app bundle (packaged builds + pnpm dev).
const appPath = computerUseHelperAppPath();
if (appPath) {
const bin = path.join(appPath, "Contents", "MacOS", COMPUTER_USE_HELPER_EXECUTABLE);
if (existsSync(bin)) return bin;
}
-
- // 3. Dev fallback — raw Swift build output.
if (!app.isPackaged) {
const swiftPkg = path.resolve(__dirname, "../../..", "packages/computer-use/native");
- const devCandidates = [
- path.join(swiftPkg, ".build", "release", "ComputerUse"),
- path.join(swiftPkg, ".build", "arm64-apple-macosx", "release", "ComputerUse"),
- path.join(swiftPkg, ".build", "debug", "ComputerUse"),
- path.join(swiftPkg, ".build", "arm64-apple-macosx", "debug", "ComputerUse"),
- ];
- for (const c of devCandidates) {
+ for (const c of [".build/release/ComputerUse", ".build/arm64-apple-macosx/release/ComputerUse", ".build/debug/ComputerUse", ".build/arm64-apple-macosx/debug/ComputerUse"].map((rel) => path.join(swiftPkg, rel))) {
if (existsSync(c)) return c;
}
}
-
return null;
}
async function checkComputerUsePermissions() {
if (process.platform !== "darwin") {
- const host = await computerUseHost();
- const available = host.availability;
- return { ok: available.supported, accessibility: available.supported, screenRecording: available.supported, supported: available.supported, protocolVersion: "omnirush.computer-use/1", platform: process.platform, modes: ["observe", "control"], backgroundControl: false, ...(available.error ? { error: available.error } : {}) };
+ const available = host?.availability ?? { supported: false, error: server?.enabled ? "Computer use is starting." : "Computer use is turned off." };
+ return { ok: available.supported, accessibility: available.supported, screenRecording: available.supported, supported: available.supported, protocolVersion: "omnirush.computer-use/2", platform: process.platform, modes: ["observe", "control"], backgroundControl: false, ...(available.error ? { error: available.error } : {}) };
}
- // Spawn binary --check → read JSON from stdout → exit. Always fresh.
const bin = resolveComputerUseExecutable();
- if (!bin) {
- return { ok: false, accessibility: false, screenRecording: false, error: "Helper binary not found. Run pnpm dev to build it." };
- }
- return spawnCheckPermissions(bin);
+ if (!bin) return { ok: false, accessibility: false, screenRecording: false, error: "Helper binary not found. Run pnpm dev to build it." };
+ return spawnJson(bin, ["--check"], (parsed) => ({
+ ok: parsed?.ok === true,
+ accessibility: parsed?.accessibility === true,
+ screenRecording: parsed?.screenRecording === true,
+ supported: parsed?.supported === true,
+ protocolVersion: parsed?.protocolVersion,
+ }), { ok: false, accessibility: false, screenRecording: false, error: "Permission check returned invalid output." });
}
-function spawnCheckPermissions(bin) {
+function spawnJson(bin, args, map, fallback) {
return new Promise((resolve) => {
let stdout = "";
- const child = spawn(bin, ["--check"], { stdio: ["ignore", "pipe", "ignore"], timeout: 5_000 });
+ const child = spawn(bin, args, { stdio: ["ignore", "pipe", "ignore"], timeout: 5_000 });
child.stdout.setEncoding("utf8");
child.stdout.on("data", (chunk) => { stdout += chunk; });
- child.on("error", () => resolve({ ok: false, accessibility: false, screenRecording: false, error: "Failed to run permission check." }));
- child.on("close", () => {
- try {
- const parsed = JSON.parse(stdout.trim());
- resolve({
- ok: parsed?.ok === true && parsed?.protocolVersion === "omnirush.computer-use/1",
- accessibility: parsed?.accessibility === true && parsed?.protocolVersion === "omnirush.computer-use/1",
- screenRecording: parsed?.screenRecording === true && parsed?.protocolVersion === "omnirush.computer-use/1",
- supported: parsed?.supported === true,
- protocolVersion: parsed?.protocolVersion,
- ...(parsed?.protocolVersion !== "omnirush.computer-use/1" ? { error: "This helper uses the previous Computer Use implementation. Rebuild or reinstall OmniRush.ai, then reconnect Computer Use." } : {}),
- });
- } catch {
- resolve({ ok: false, accessibility: false, screenRecording: false, error: "Permission check returned invalid output." });
- }
- });
+ child.on("error", () => resolve(fallback));
+ child.on("close", () => { try { resolve(map(JSON.parse(stdout.trim()))); } catch { resolve(fallback); } });
});
}
async function listRunningApps() {
- // Spawn binary --list-apps → read JSON from stdout → exit. Needs no TCC
- // permissions, so this works before Computer Use setup is complete.
- if (process.platform !== "darwin") return (await computerUseHost()).apps();
+ if (process.platform !== "darwin") {
+ if (!host) return { ok: false, apps: [] };
+ const connection = host.connect();
+ try {
+ const reply = await connection.handle({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "computer_discover", arguments: {} } });
+ const state = JSON.parse(reply?.result?.content?.[0]?.text ?? "{}");
+ return { ok: state.ok === true, apps: [...new Set((state.apps ?? []).filter((a) => a.windows?.length).map((a) => a.name))] };
+ } finally { await connection.close(); }
+ }
const bin = resolveComputerUseExecutable();
if (!bin) return { ok: false, apps: [] };
- return new Promise((resolve) => {
- let stdout = "";
- const child = spawn(bin, ["--list-apps"], { stdio: ["ignore", "pipe", "ignore"], timeout: 5_000 });
- child.stdout.setEncoding("utf8");
- child.stdout.on("data", (chunk) => { stdout += chunk; });
- child.on("error", () => resolve({ ok: false, apps: [] }));
- child.on("close", () => {
- try {
- const parsed = JSON.parse(stdout.trim());
- const apps = Array.isArray(parsed?.apps) ? parsed.apps.filter((name) => typeof name === "string" && name.trim()) : [];
- resolve({ ok: parsed?.ok === true, apps });
- } catch {
- resolve({ ok: false, apps: [] });
- }
- });
- });
+ return spawnJson(bin, ["--list-apps"], (parsed) => ({ ok: parsed?.ok === true, apps: Array.isArray(parsed?.apps) ? parsed.apps.filter((name) => typeof name === "string" && name.trim()) : [] }), { ok: false, apps: [] });
}
let setupProcess = null;
async function openComputerUseSetupApp() {
- if (process.platform !== "darwin") { await computerUseHost(); return; }
+ if (process.platform !== "darwin") return;
const bin = resolveComputerUseExecutable();
if (!bin) throw new Error("The Computer Use helper is unavailable. Rebuild or reinstall OmniRush.ai.");
- // Keep the responsible application consistent with --check and the MCP
- // child. LaunchServices gives the GUI its own TCC identity instead.
- if (setupProcess && setupProcess.exitCode === null && !setupProcess.killed) {
- setupProcess.kill("SIGUSR1");
- return;
- }
+ // Keep the responsible application consistent with --check and the MCP child.
+ if (setupProcess && setupProcess.exitCode === null && !setupProcess.killed) { setupProcess.kill("SIGUSR1"); return; }
const child = spawn(bin, ["setup"], { stdio: "ignore" });
setupProcess = child;
child.once("exit", () => { if (setupProcess === child) setupProcess = null; });
await new Promise((resolve, reject) => {
child.once("spawn", resolve);
- child.once("error", (error) => {
- if (setupProcess === child) setupProcess = null;
- reject(error);
- });
+ child.once("error", (error) => { if (setupProcess === child) setupProcess = null; reject(error); });
});
child.unref();
}
export {
getComputerUseState,
+ computerUseAnswer,
+ computerUseControl,
getComputerUseMcpEnvironment,
- computerUseAction,
checkComputerUsePermissions,
getComputerUseMcpCommand,
listRunningApps,
diff --git a/apps/desktop/electron/main.mjs b/apps/desktop/electron/main.mjs
index 765adb77..0505a66f 100644
--- a/apps/desktop/electron/main.mjs
+++ b/apps/desktop/electron/main.mjs
@@ -28,7 +28,9 @@ import {
getComputerUseMcpCommand,
getComputerUseMcpEnvironment,
getComputerUseState,
- computerUseAction,
+ computerUseAnswer,
+ computerUseControl,
+ initComputerUse,
listRunningApps,
openComputerUseSetupApp,
} from "./computer-use.mjs";
@@ -2311,9 +2313,14 @@ const desktopCommandHandlers = {
return omnirushUiMcpLaunch().command;
},
"getComputerUseState": async () => getComputerUseState(),
- "computerUseAction": async (event, value) => {
- if (!mainWindow || event.sender !== mainWindow.webContents || event.senderFrame !== mainWindow.webContents.mainFrame) throw new Error("Computer Use controls require the main OmniRush.ai window.");
- return computerUseAction(value);
+ // Only the person answers, in the main window's own frame (never the agent, a page or a config).
+ "computerUseAnswer": async (event, value) => {
+ if (!mainWindow || event.sender !== mainWindow.webContents || event.senderFrame !== mainWindow.webContents.mainFrame) throw new Error("Computer use controls require the main OmniRush.ai window.");
+ return computerUseAnswer(value);
+ },
+ "computerUseControl": async (event, value) => {
+ if (!mainWindow || event.sender !== mainWindow.webContents || event.senderFrame !== mainWindow.webContents.mainFrame) throw new Error("Computer use controls require the main OmniRush.ai window.");
+ return computerUseControl(value);
},
"getComputerUseMcpCommand": async (event, ...args) => {
return getComputerUseMcpCommand();
@@ -3443,6 +3450,8 @@ or use: pnpm dev:worktree`);
});
app.whenReady().then(async () => {
+ // Computer use follows the local server's switch (computer-use.mjs); the host starts only when it is on.
+ initComputerUse({ getWindow: () => mainWindow, serverInfo: () => runtimeManager.omnirushServerInfo(), log: (message, error) => console.warn(`[computer-use] ${message}`, error?.message ?? "") });
const systemCaCertificates = await runtimeManager.systemCaCertificates();
session.defaultSession.setCertificateVerifyProc(createSystemCaCertificateVerifyProc(systemCaCertificates));
installMediaPermissionHandlers(session, () => mainWindow);
diff --git a/apps/desktop/electron/preload.mjs b/apps/desktop/electron/preload.mjs
index 979554f5..44aebd83 100644
--- a/apps/desktop/electron/preload.mjs
+++ b/apps/desktop/electron/preload.mjs
@@ -91,6 +91,16 @@ contextBridge.exposeInMainWorld("__OMNIRUSH_ELECTRON__", {
return detail;
},
},
+ computerUse: {
+ /** Computer use status (asks, sessions, frames) from the host in Electron main. */
+ onStatus(callback) {
+ const handler = (_event, status) => callback(status);
+ ipcRenderer.on("omnirush:computer-use:status", handler);
+ return () => {
+ ipcRenderer.removeListener("omnirush:computer-use:status", handler);
+ };
+ },
+ },
automationRunner: {
onCredentialRejected(callback) {
const handler = () => callback();
diff --git a/apps/server/scripts/copy-shared-modules.mjs b/apps/server/scripts/copy-shared-modules.mjs
index b2c43353..34b9a9e3 100644
--- a/apps/server/scripts/copy-shared-modules.mjs
+++ b/apps/server/scripts/copy-shared-modules.mjs
@@ -1,11 +1,11 @@
// tsc emits only TypeScript: the plain-JavaScript modules shared with the CLI
-// (src/workspace/*.mjs, see PARITY.sha256 there) are copied into dist as they are.
+// (src/workspace/*.mjs and src/computer-use/shared/*.mjs, see PARITY.sha256 there) are copied into dist as they are.
import { copyFileSync, mkdirSync, readdirSync } from "node:fs";
import { join, resolve } from "node:path";
const root = resolve(import.meta.dirname, "..");
const copied = [];
-for (const folder of ["workspace"]) {
+for (const folder of ["workspace", "computer-use/shared"]) {
const from = join(root, "src", folder);
const to = join(root, "dist", folder);
mkdirSync(to, { recursive: true });
diff --git a/apps/server/src/capture-host.ts b/apps/server/src/capture-host.ts
index f457bd2d..3559793b 100644
--- a/apps/server/src/capture-host.ts
+++ b/apps/server/src/capture-host.ts
@@ -8,6 +8,7 @@
* same behaviour. Every call takes and returns plain data, so the same calls
* cross the thread boundary unchanged.
*/
+import { createMediaRewriter } from "./computer-use/media-capture.js";
import type { ApprovalMode } from "./approval-mode.js";
import type { ContextOptions } from "./context/index.js";
import { readPromptAttachments, promptBodyForTrace, v2PromptBodyForTrace } from "./session-upload-attachments.js";
@@ -17,6 +18,7 @@ import {
currentEngineTarget,
engineReplaced,
observeUploadedSession,
+ setEngineMessageRewrite,
followToolStart, followContextToolEvents, followFilesUsedCalls,
projectArchiveEngineReads,
promptDispatched,
@@ -218,6 +220,12 @@ export class CaptureHost {
...(baseMaxDeferMs !== undefined ? { baseMaxDeferMs } : {}),
});
this.archive.start();
+ // Image data URLs in tool results become media references as messages are read (computer-use/media-capture.ts).
+ setEngineMessageRewrite(createMediaRewriter({
+ stateDir: options.stateDir,
+ onFiles: (sessionId, files) => this.archive.mediaFiles(sessionId, files),
+ onComputerUse: (sessionId) => this.sessionUploader.markComputerUse(sessionId),
+ }));
}
startSession(sessionId: string, workspaceId: string, root: string, turnsBefore = 0): void {
diff --git a/apps/server/src/computer-use/desktop.test.ts b/apps/server/src/computer-use/desktop.test.ts
new file mode 100644
index 00000000..64363042
--- /dev/null
+++ b/apps/server/src/computer-use/desktop.test.ts
@@ -0,0 +1,148 @@
+// Desktop computer use on the server side: the switch, the built-in MCP entry
+// (reserved id), the engine plugin's hooks, the media lane and its store.
+import { afterAll, describe, expect, test } from "bun:test";
+import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import omnirushMcpPolicy from "../opencode-plugins/omnirush-mcp-policy.js";
+import { MediaStore } from "../session-archive/media-store.js";
+import { computerUseBuiltins, computerUseDir, computerUseSettingFor, createChatTracker, permissionModeName, rememberComputerUseSwitch, sessionEventOf, writeHostFile } from "./desktop.js";
+import { createMediaRewriter, usesComputer } from "./media-capture.js";
+import { encodePng } from "./shared/image.mjs";
+
+const dirs: string[] = [];
+function tmp() {
+ const dir = mkdtempSync(path.join(os.tmpdir(), "cu-desk-"));
+ dirs.push(dir);
+ return dir;
+}
+afterAll(() => { for (const dir of dirs) rmSync(dir, { recursive: true, force: true }); });
+
+function png(seed: number) {
+ const data = new Uint8Array(32 * 20 * 4).fill(seed);
+ for (let i = 3; i < data.length; i += 4) data[i] = 255;
+ data[0] = seed; data[1] = seed >> 8;
+ return encodePng({ width: 32, height: 20, data });
+}
+
+describe("switch and built-in entry", () => {
+ test("remembered block; env override; the built-in only with a running host; id always reserved", () => {
+ const state = tmp();
+ const dir = computerUseDir(state);
+ expect(computerUseSettingFor(dir, {}).enabled).toBe(false);
+ expect(rememberComputerUseSwitch(state, { computer_use: { enabled: true, limits: { actions_per_session: 9 } } })).toBe(true);
+ expect(rememberComputerUseSwitch(state, { computer_use: { enabled: true, limits: { actions_per_session: 9 } } })).toBe(false);
+ expect(computerUseSettingFor(dir, {}).limits.actions_per_session).toBe(9);
+ expect(computerUseSettingFor(dir, { OMNIRUSH_COMPUTER_USE: "0" }).enabled).toBe(false);
+ expect(computerUseBuiltins(dir, {})).toEqual({ "computer-use": null });
+ expect(writeHostFile(dir, { command: [process.execPath, "/x/relay.mjs", "/x/connection.json"], environment: { ELECTRON_RUN_AS_NODE: "1" } })).toBe(true);
+ expect(computerUseBuiltins(dir, {})["computer-use"]).toMatchObject({ type: "local", command: [process.execPath, "/x/relay.mjs", "/x/connection.json"], environment: { ELECTRON_RUN_AS_NODE: "1" }, enabled: true, timeout: 300_000 });
+ expect(rememberComputerUseSwitch(state, {})).toBe(true);
+ expect(computerUseBuiltins(dir, {})).toEqual({ "computer-use": null });
+ expect(computerUseBuiltins(dir, { OMNIRUSH_COMPUTER_USE: "1" })["computer-use"]).not.toBeNull();
+ expect(writeHostFile(dir, null)).toBe(true);
+ expect(computerUseBuiltins(undefined, {})).toEqual({ "computer-use": null });
+ });
+
+ test("permission mode names, chat tracker, event checks", () => {
+ expect([permissionModeName("careful"), permissionModeName("guarded"), permissionModeName("full"), permissionModeName(null)]).toEqual(["careful", "normal", "auto", "normal"]);
+ const chats = createChatTracker();
+ chats.mark("ses_a"); chats.mark("ses_b"); chats.mark("ses_a");
+ expect(chats.resolve(null)).toBe("ses_a");
+ expect(chats.resolve(["ses_b"])).toBe("ses_b");
+ expect(sessionEventOf({ event: "opened", by: "agent", app_id: "/usr/bin/gedit", screen: { id: "0", width: 10, height: 5, scale: 1 }, junk: 1 })).toEqual({ event: "opened", by: "agent", app_id: "/usr/bin/gedit", screen: { id: "0", width: 10, height: 5, scale: 1 } });
+ expect(sessionEventOf({ event: "hacked", by: "agent" })).toBeNull();
+ });
+});
+
+describe("engine MCP plugin", () => {
+ async function run(env: Record<string, string>, config: Record<string, unknown>) {
+ const saved = { ...process.env };
+ Object.assign(process.env, env);
+ try {
+ const plugin = await omnirushMcpPolicy({ directory: tmp() });
+ await plugin.config(config);
+ return { config, plugin };
+ } finally {
+ for (const key of Object.keys(env)) delete process.env[key];
+ Object.assign(process.env, saved);
+ }
+ }
+
+ test("a config entry named computer-use is dropped and reported; the built-in is added with no sub-agent access", async () => {
+ const state = tmp();
+ const dir = computerUseDir(state);
+ const reports = path.join(state, "reports");
+ writeHostFile(dir, { command: [process.execPath, "/x/relay.mjs", "/x/c.json"], environment: {} });
+ const { config } = await run({ OMNIRUSH_COMPUTER_USE_DIR: dir, OMNIRUSH_COMPUTER_USE: "1", OMNIRUSH_MCP_REPORT_DIR: reports }, {
+ mcp: { "computer-use": { type: "local", command: ["npx", "-y", "evil"] } },
+ permission: { "*": "ask" },
+ agent: { custom: { mode: "subagent" } },
+ });
+ const mcp = config.mcp as Record<string, { command: string[] }>;
+ expect(Object.keys(mcp)).toEqual(["computer-use"]);
+ expect(mcp["computer-use"]?.command).toEqual([process.execPath, "/x/relay.mjs", "/x/c.json"]);
+ expect(Object.entries(config.permission as Record<string, string>).at(-1)).toEqual(["computer-use_*", "allow"]);
+ const agents = config.agent as Record<string, { permission: Record<string, string> }>;
+ for (const name of ["general", "explore", "custom"]) expect(agents[name]?.permission["computer-use_*"]).toBe("deny");
+ const report = JSON.parse(readFileSync(path.join(reports, readdirName(reports)), "utf8")) as { decisions: Array<{ name: string; code: string }> };
+ expect(report.decisions.find((d) => d.name === "computer-use")).toMatchObject({ code: "reserved" });
+ });
+
+ test("off: nothing is added and the config is as before", async () => {
+ const state = tmp();
+ const { config } = await run({ OMNIRUSH_COMPUTER_USE_DIR: computerUseDir(state) }, { permission: { "*": "ask" } });
+ expect(config).toEqual({ mcp: {}, permission: { "*": "ask" } });
+ });
+
+ test("model context pruning touches only computer tool parts", async () => {
+ const { plugin } = await run({}, {});
+ const url = `data:image/png;base64,${png(1).toString("base64")}`;
+ const part = (tool: string, i: number) => ({ type: "tool", tool, state: { status: "completed", output: "{}", attachments: [{ mime: "image/png", url: url + "#".repeat(i % 2) }] } });
+ const computer = Array.from({ length: 13 }, (_, i) => part("computer-use_computer_act", i));
+ const playwright = Array.from({ length: 13 }, (_, i) => part("playwright_browser_take_screenshot", i));
+ const output = { messages: [{ info: {}, parts: [...computer, ...playwright] }] };
+ await plugin["experimental.chat.messages.transform"]({}, output);
+ expect(computer.filter((p) => p.state.attachments.length).length).toBe(5);
+ expect(playwright.filter((p) => p.state.attachments.length).length).toBe(13);
+ });
+});
+
+function readdirName(dir: string): string {
+ const name = readdirSync(dir).find((n) => n.endsWith(".json"));
+ if (!name) throw new Error("no report");
+ return name;
+}
+
+describe("media lane", () => {
+ test("rewriter: refs, archive files once, computer use noticed; messages without images unchanged", () => {
+ const state = tmp();
+ const files: Array<{ sessionId: string; paths: string[] }> = [];
+ const used: string[] = [];
+ const rewrite = createMediaRewriter({ stateDir: state, onFiles: (sessionId, list) => files.push({ sessionId, paths: list.map((f) => f.path) }), onComputerUse: (sessionId) => used.push(sessionId) });
+ const plain = { info: { id: "m1" }, parts: [{ type: "text", text: "hi" }] };
+ expect(rewrite("ses_1", plain)).toBe(plain);
+ const message = { info: { id: "m2" }, parts: [{ type: "tool", tool: "computer-use_computer_observe", state: { status: "completed", output: "{}", attachments: [{ type: "file", mime: "image/png", url: `data:image/png;base64,${png(7).toString("base64")}` }] } }] };
+ const out = rewrite("ses_1", message) as typeof message;
+ expect(out.parts[0]?.state.attachments[0]?.url).toMatch(/^omnirush-media:[a-f0-9]{64}\.png$/);
+ expect(files).toEqual([{ sessionId: "ses_1", paths: [expect.stringMatching(/^__media__\/[a-f0-9]{64}\.png$/)] }]);
+ expect(used).toEqual(["ses_1"]);
+ rewrite("ses_1", message);
+ expect(files.length).toBe(1);
+ expect(usesComputer(plain)).toBe(false);
+ });
+
+ test("media store: staged once per path, listed while in the spool", async () => {
+ const dir = tmp();
+ const store = new MediaStore(path.join(dir, "media"));
+ const file = path.join(dir, "a.png");
+ writeFileSync(file, png(3));
+ const record = { path: `__media__/${"a".repeat(64)}.png`, file, sha: "a".repeat(64), kind: "model" as const, bytes: 10 };
+ expect(await store.add("ses_x", [record, record, { ...record, path: "../escape" }])).toBe(1);
+ expect(await store.add("ses_x", [record])).toBe(0);
+ expect((await store.present("ses_x")).map((r) => r.path)).toEqual([record.path]);
+ rmSync(file);
+ expect(await store.present("ses_x")).toEqual([]);
+ });
+});
diff --git a/apps/server/src/computer-use/desktop.ts b/apps/server/src/computer-use/desktop.ts
new file mode 100644
index 00000000..e57bd8c6
--- /dev/null
+++ b/apps/server/src/computer-use/desktop.ts
@@ -0,0 +1,170 @@
+/**
+ * Computer use in the desktop app: the server switch (GET /collect/capabilities
+ * `computer_use`, remembered like web search), the host file Electron main
+ * writes when its host listens, the built-in MCP entry the engine's MCP plugin
+ * adds (id `computer-use`, reserved), and where frames are spooled.
+ *
+ * Files, under <runtimeStorageDir>/computer-use/ (OMNIRUSH_COMPUTER_USE_DIR in
+ * the engine's environment):
+ * server.json the switch as last seen
+ * host.json { command, environment } of the running host (Electron main)
+ * media-spool/ frames the host took (media.mjs createMediaSpool)
+ * media-index/ per-chat media index (media.mjs createMediaIndex)
+ *
+ * On, in this order: OMNIRUSH_COMPUTER_USE=1/0 in the environment, then the
+ * server's block, else off (policy.mjs computerUseSetting).
+ */
+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { SERVER_ID } from "./shared/contract.mjs";
+import { computerUseSetting, type ComputerUseSetting } from "./shared/policy.mjs";
+
+export const COMPUTER_USE_DIR_ENV = "OMNIRUSH_COMPUTER_USE_DIR";
+/** How long an MCP call may take (an open-session waits for the person). */
+export const COMPUTER_USE_MCP_TIMEOUT_MS = 300_000;
+
+export type ComputerUseHostFile = { command: string[]; environment: Record<string, string> };
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function readJson(file: string): unknown {
+ try {
+ return JSON.parse(readFileSync(file, "utf8"));
+ } catch {
+ return null;
+ }
+}
+
+function writeJson(file: string, value: unknown): void {
+ mkdirSync(join(file, ".."), { recursive: true, mode: 0o700 });
+ const temp = `${file}.${process.pid}.tmp`;
+ writeFileSync(temp, JSON.stringify(value), { mode: 0o600 });
+ renameSync(temp, file);
+}
+
+export function computerUseDir(stateDir: string): string {
+ return join(stateDir, "computer-use");
+}
+export function mediaSpoolDir(stateDir: string): string {
+ return join(computerUseDir(stateDir), "media-spool");
+}
+export function mediaIndexDir(stateDir: string): string {
+ return join(computerUseDir(stateDir), "media-index");
+}
+
+/** The engine's environment for its plugins. */
+export function computerUseEngineEnv(stateDir: string): Record<string, string> {
+ return { [COMPUTER_USE_DIR_ENV]: computerUseDir(stateDir) };
+}
+
+/** The block as last seen (null: never seen). */
+export function rememberedComputerUseBlock(dir: string): Record<string, unknown> | null {
+ const value = readJson(join(dir, "server.json"));
+ return isRecord(value) && isRecord(value.block) ? value.block : null;
+}
+
+/**
+ * Remembers the `computer_use` block of a /collect/capabilities answer (a
+ * missing block is off; null, no answer, keeps the last). Whether it changed.
+ */
+export function rememberComputerUseSwitch(stateDir: string, capabilities: unknown): boolean {
+ if (!stateDir || !isRecord(capabilities)) return false;
+ const dir = computerUseDir(stateDir);
+ const block = isRecord(capabilities.computer_use) ? capabilities.computer_use : null;
+ const before = readJson(join(dir, "server.json"));
+ const beforeBlock = isRecord(before) && "block" in before ? before.block : undefined;
+ try {
+ writeJson(join(dir, "server.json"), { block, at: Date.now() });
+ } catch {
+ return false;
+ }
+ return JSON.stringify(beforeBlock ?? null) !== JSON.stringify(block);
+}
+
+/** Computer use for this desktop now. */
+export function computerUseSettingFor(dir: string, env: Record<string, string | undefined> = process.env, platform = process.platform): ComputerUseSetting {
+ return computerUseSetting({ env, block: rememberedComputerUseBlock(dir) ?? undefined, client: "desktop", platform });
+}
+
+/** The running host's MCP command, as Electron main wrote it. */
+export function readHostFile(dir: string): ComputerUseHostFile | null {
+ const value = readJson(join(dir, "host.json"));
+ if (!isRecord(value) || !Array.isArray(value.command) || !value.command.every((part) => typeof part === "string") || value.command.length < 1) return null;
+ const environment = isRecord(value.environment) ? Object.fromEntries(Object.entries(value.environment).filter((entry): entry is [string, string] => typeof entry[1] === "string")) : {};
+ return { command: value.command, environment };
+}
+
+/** Writes (or, with null, removes) the host file. Whether it changed. */
+export function writeHostFile(dir: string, host: ComputerUseHostFile | null): boolean {
+ const before = JSON.stringify(readHostFile(dir));
+ if (host) writeJson(join(dir, "host.json"), host);
+ else rmSync(join(dir, "host.json"), { force: true });
+ return before !== JSON.stringify(host ? readHostFile(dir) : null);
+}
+
+/**
+ * The built-in servers for the MCP plugin: `computer-use` when it is on and
+ * the host runs; the id stays reserved (null) otherwise, so a config entry
+ * with that name is always left out.
+ */
+export function computerUseBuiltins(dir: string | null | undefined, env: Record<string, string | undefined> = process.env): Record<string, Record<string, unknown> | null> {
+ if (!dir) return { [SERVER_ID]: null };
+ const setting = computerUseSettingFor(dir, env);
+ const host = setting.enabled && setting.real_screen ? readHostFile(dir) : null;
+ if (!host || !existsSync(host.command[0] ?? "")) return { [SERVER_ID]: null };
+ return { [SERVER_ID]: { type: "local", command: host.command, environment: host.environment, enabled: true, timeout: COMPUTER_USE_MCP_TIMEOUT_MS, title: "Computer use" } };
+}
+
+/** The desktop's approval mode as the shared rules name it. */
+export function permissionModeName(mode: string | null | undefined): "careful" | "normal" | "auto" {
+ return mode === "careful" ? "careful" : mode === "full" || mode === "auto" ? "auto" : "normal";
+}
+
+/**
+ * The chat a computer use call belongs to: the latest prompted chat that is
+ * still running (MCP calls carry no chat id). `mark` at every prompt
+ * dispatch; `resolve` with the engine's running sessions.
+ */
+export function createChatTracker(limit = 64) {
+ const order: string[] = [];
+ return {
+ mark(sessionId: string): void {
+ const at = order.indexOf(sessionId);
+ if (at >= 0) order.splice(at, 1);
+ order.push(sessionId);
+ if (order.length > limit) order.splice(0, order.length - limit);
+ },
+ resolve(running: string[] | null): string | null {
+ for (let i = order.length - 1; i >= 0; i -= 1) {
+ const id = order[i];
+ if (id && (!running || running.includes(id))) return id;
+ }
+ return order.at(-1) ?? null;
+ },
+ };
+}
+
+/** The `environment.computer_use` envelope field of a chat that used computer use. */
+export type ComputerUseEnvironment = { enabled: true; host: "native" | "virtual"; os: string; screens: Array<{ id: string; width: number; height: number; scale: number }> };
+
+/** A `computer.session` event from the host, checked before it reaches a trace. */
+export function sessionEventOf(value: unknown): Record<string, unknown> | null {
+ if (!isRecord(value)) return null;
+ const events = ["opened", "approved", "denied", "paused", "continued", "stopped", "closed", "expired"];
+ const by = ["person", "agent", "limit", "server"];
+ if (typeof value.event !== "string" || !events.includes(value.event) || typeof value.by !== "string" || !by.includes(value.by)) return null;
+ const out: Record<string, unknown> = { event: value.event, by: value.by };
+ for (const key of ["session_id", "app_id", "app_name", "bundle_or_exe", "mode", "approval", "os", "reason"]) {
+ if (typeof value[key] === "string" && String(value[key]).length <= 4096) out[key] = value[key];
+ }
+ if (value.reused === true) out.reused = true;
+ if (isRecord(value.screen)) {
+ const s = value.screen;
+ out.screen = { id: String(s.id ?? ""), width: Number(s.width) || 0, height: Number(s.height) || 0, scale: Number(s.scale) || 1 };
+ }
+ if (isRecord(value.virtual)) out.virtual = value.virtual;
+ return out;
+}
diff --git a/apps/server/src/computer-use/media-capture.ts b/apps/server/src/computer-use/media-capture.ts
new file mode 100644
index 00000000..406ee1bb
--- /dev/null
+++ b/apps/server/src/computer-use/media-capture.ts
@@ -0,0 +1,80 @@
+/**
+ * The uploader's media lane: every engine message the observer reads goes
+ * through `rewrite` (session-upload-observer.ts, part by part, before the
+ * turn's size budgets), which replaces image data URLs in tool results with
+ * `omnirush-media:` references (shared/media.mjs) and hands the frames to the
+ * project archive (`__media__/`, its own caps). A message without images is
+ * returned as it is (the same object).
+ */
+import { join } from "node:path";
+
+import { isComputerTool } from "./shared/contract.mjs";
+import { createMediaIndex, createMediaSpool, rewriteMessageMedia, type MediaFile, type MediaIndex } from "./shared/media.mjs";
+import { computerUseDir, computerUseSettingFor, mediaIndexDir, mediaSpoolDir } from "./desktop.js";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/** A file-name-safe key of a session id. */
+function keyOf(sessionId: string): string {
+ return sessionId.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
+}
+
+/** Whether a message has a tool part of the built-in computer use server. */
+export function usesComputer(message: unknown): boolean {
+ return isRecord(message) && Array.isArray(message.parts) && message.parts.some((part) => isRecord(part) && part.type === "tool" && isComputerTool(part.tool));
+}
+
+function hasImageAttachment(message: unknown): boolean {
+ if (!isRecord(message) || !Array.isArray(message.parts)) return false;
+ return message.parts.some((part) => isRecord(part) && part.type === "tool" && isRecord(part.state) && Array.isArray(part.state.attachments)
+ && part.state.attachments.some((a) => isRecord(a) && typeof a.url === "string" && a.url.startsWith("data:image/")));
+}
+
+export type MediaRewriter = (sessionId: string, message: unknown) => unknown;
+
+export function createMediaRewriter(options: {
+ stateDir: string;
+ onFiles: (sessionId: string, files: MediaFile[]) => void;
+ onComputerUse: (sessionId: string) => void;
+ log?: (message: string, detail?: Record<string, unknown>) => void;
+}): MediaRewriter {
+ const spool = createMediaSpool(mediaSpoolDir(options.stateDir));
+ const indexes = new Map<string, MediaIndex>();
+ const indexOf = (sessionId: string) => {
+ let index = indexes.get(sessionId);
+ if (!index) {
+ index = createMediaIndex(join(mediaIndexDir(options.stateDir), `${keyOf(sessionId)}.json`));
+ indexes.set(sessionId, index);
+ while (indexes.size > 256) indexes.delete(indexes.keys().next().value as string);
+ }
+ return index;
+ };
+ return (sessionId, message) => {
+ if (usesComputer(message)) options.onComputerUse(sessionId);
+ if (!hasImageAttachment(message)) return message;
+ try {
+ const limits = computerUseSettingFor(computerUseDir(options.stateDir)).limits;
+ const out = rewriteMessageMedia(message, { spool, index: indexOf(sessionId), limits, chat: sessionId });
+ if (out.files.length) options.onFiles(sessionId, out.files);
+ return out.message;
+ } catch (error) {
+ // Fail closed for the size caps: the images leave the message, the turn still uploads.
+ options.log?.("media references failed; images left out of the message", { error: error instanceof Error ? error.message : String(error) });
+ return dropImages(message);
+ }
+ };
+}
+
+function dropImages(message: unknown): unknown {
+ if (!isRecord(message) || !Array.isArray(message.parts)) return message;
+ return {
+ ...message,
+ parts: message.parts.map((part) => {
+ if (!isRecord(part) || !isRecord(part.state) || !Array.isArray(part.state.attachments)) return part;
+ const attachments = part.state.attachments.map((a) => (isRecord(a) && typeof a.url === "string" && a.url.startsWith("data:image/") ? { ...a, url: null, omitted: "media_error" } : a));
+ return { ...part, state: { ...part.state, attachments } };
+ }),
+ };
+}
diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.ts
index 88114191..463c9b3f 100644
--- a/apps/server/src/computer-use/portable-host.test.ts
+++ b/apps/server/src/computer-use/portable-host.test.ts
@@ -6,7 +6,7 @@ import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod
import os from "node:os";
import path from "node:path";
-import { createPortableHost } from "./shared/portable-host.mjs";
+import { createPortableHost, type ActRequest, type OpenRequest } from "./shared/portable-host.mjs";
import { createMediaSpool } from "./shared/media.mjs";
type Window = { id: number; pid: number; identity: string; executable: string; appId: string; appName: string; title: string; bounds: { x: number; y: number; width: number; height: number }; dpi: number; scale: number; monitor: string };
@@ -34,6 +34,7 @@ function fakeNative() {
const native = {
ready: Promise.resolve({ supported: true }),
pid: 999_999,
+ held: { keys: [], buttons: [] },
calls,
state,
windows,
@@ -64,8 +65,8 @@ function rig(mode: "careful" | "normal" | "auto", options: { decision?: "chat" |
const dir = mkdtempSync(path.join(os.tmpdir(), "cu-host-"));
dirs.push(dir);
const events: Array<Record<string, unknown>> = [];
- const opens: Array<Record<string, unknown>> = [];
- const acts: Array<Record<string, unknown>> = [];
+ const opens: OpenRequest[] = [];
+ const acts: ActRequest[] = [];
const settings = { enabled: options.enabled ?? true, real_screen: true, limits: options.limits ?? {}, deny_apps: [], mode };
const hostPromise = createPortableHost({
native,
@@ -73,8 +74,8 @@ function rig(mode: "careful" | "normal" | "auto", options: { decision?: "chat" |
settings: () => settings,
context: () => ({ ancestors: [], chat: "ses_1" }),
approve: {
- open: async (request: Record<string, unknown>) => { opens.push(request); return { decision: options.decision ?? "once", window_id: request.window_id }; },
- act: async (request: Record<string, unknown>) => { acts.push(request); return { allow: options.allowAct ?? true }; },
+ open: async (request: OpenRequest) => { opens.push(request); return { decision: options.decision ?? "once", window_id: request.window_id }; },
+ act: async (request: ActRequest) => { acts.push(request); return { allow: options.allowAct ?? true }; },
},
onEvent: (event: Record<string, unknown>) => events.push(event),
});
@@ -187,7 +188,7 @@ describe("portable host: observe and act", () => {
const act = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Type a greeting", action: { type: "type", text: "hello" } });
expect(act.state).toMatchObject({ ok: false, code: "approval_denied", approval: { asked: true, reasons: ["careful"] } });
expect(r.acts[0]).toMatchObject({ intent: "Type a greeting", action: { type: "type", text: "hello" }, app: { app_name: "gedit" } });
- expect(String((r.acts[0]?.frame as { data_url: string }).data_url)).toStartWith("data:image/");
+ expect(String(r.acts[0]?.frame.data_url)).toStartWith("data:image/");
expect(r.native.calls.filter((x) => x.method === "act").length).toBe(0);
await host.close();
});
diff --git a/apps/server/src/computer-use/routes.ts b/apps/server/src/computer-use/routes.ts
new file mode 100644
index 00000000..b364d2ab
--- /dev/null
+++ b/apps/server/src/computer-use/routes.ts
@@ -0,0 +1,91 @@
+/**
+ * The desktop's computer use routes, for Electron main only (client scope,
+ * loopback): the switch and permission mode it applies, the host file it
+ * writes when its host listens, and the `computer.session` events it reports.
+ *
+ * GET /computer-use/state { enabled, setting, mode, dir, spool, chat }
+ * POST /computer-use/refresh reads /collect/capabilities again (≤ 60 s old), then the state
+ * POST /computer-use/host { command, environment } | { host: null }
+ * POST /computer-use/event a computer.session event → the chat's trace
+ */
+import { addRoute, type Route } from "../routes/registry.js";
+import {
+ computerUseDir,
+ computerUseSettingFor,
+ type createChatTracker,
+ mediaSpoolDir,
+ permissionModeName,
+ readHostFile,
+ rememberComputerUseSwitch,
+ sessionEventOf,
+ writeHostFile,
+} from "./desktop.js";
+
+export type ComputerUseRouteDeps = {
+ routes: Route[];
+ stateDir: () => string;
+ jsonResponse: (data: unknown, status?: number) => Response;
+ readJsonBody: (request: Request) => Promise<Record<string, unknown>>;
+ /** The desktop approval mode ("careful" | "guarded" | "full"), or null. */
+ approvalMode: () => Promise<string | null>;
+ /** Fresh capabilities (strict), or throws. */
+ readCapabilities: (maxAgeMs: number) => Promise<unknown>;
+ /** Restart the engine's projects so they pick up the built-in server. */
+ applyEngine: () => Promise<unknown>;
+ chats: ReturnType<typeof createChatTracker>;
+ /** The engine's running sessions, or null when unknown. */
+ runningSessions: () => Promise<string[] | null>;
+ /** Records a trace event on a chat the uploader has open. */
+ recordTrace: (sessionId: string, type: string, data: Record<string, unknown>) => void;
+};
+
+export function registerComputerUseRoutes(deps: ComputerUseRouteDeps): void {
+ const { routes, jsonResponse } = deps;
+ const state = async () => {
+ const dir = computerUseDir(deps.stateDir());
+ const setting = computerUseSettingFor(dir);
+ const mode = permissionModeName(await deps.approvalMode().catch(() => null));
+ const chat = deps.chats.resolve(await deps.runningSessions().catch(() => null));
+ return {
+ enabled: setting.enabled,
+ setting: { enabled: setting.enabled, real_screen: setting.real_screen, virtual_display: setting.virtual_display, limits: setting.limits, deny_apps: setting.deny_apps, models: setting.models, message: setting.message, reason: setting.reason, source: setting.source },
+ mode,
+ dir,
+ spool: mediaSpoolDir(deps.stateDir()),
+ host: readHostFile(dir) !== null,
+ chat,
+ };
+ };
+
+ addRoute(routes, "GET", "/computer-use/state", "client", async () => jsonResponse(await state()));
+
+ addRoute(routes, "POST", "/computer-use/refresh", "client", async () => {
+ try {
+ const capabilities = await deps.readCapabilities(60_000);
+ if (rememberComputerUseSwitch(deps.stateDir(), capabilities)) void deps.applyEngine().catch(() => undefined);
+ } catch {
+ // A failed read keeps the last answer.
+ }
+ return jsonResponse(await state());
+ });
+
+ addRoute(routes, "POST", "/computer-use/host", "client", async (ctx) => {
+ const body = await deps.readJsonBody(ctx.request);
+ const command = Array.isArray(body.command) && body.command.every((part) => typeof part === "string") ? (body.command as string[]) : null;
+ const environment = typeof body.environment === "object" && body.environment !== null && !Array.isArray(body.environment)
+ ? Object.fromEntries(Object.entries(body.environment as Record<string, unknown>).filter((entry): entry is [string, string] => typeof entry[1] === "string"))
+ : {};
+ const changed = writeHostFile(computerUseDir(deps.stateDir()), command && command.length ? { command, environment } : null);
+ if (changed) void deps.applyEngine().catch(() => undefined);
+ return jsonResponse({ ok: true, changed });
+ });
+
+ addRoute(routes, "POST", "/computer-use/event", "client", async (ctx) => {
+ const body = await deps.readJsonBody(ctx.request);
+ const event = sessionEventOf(body.event);
+ if (!event) return jsonResponse({ ok: false, error: "invalid_event" }, 400);
+ const chat = typeof body.chat === "string" && /^[A-Za-z0-9_-]{1,128}$/.test(body.chat) ? body.chat : deps.chats.resolve(await deps.runningSessions().catch(() => null));
+ if (chat) deps.recordTrace(chat, "computer.session", event);
+ return jsonResponse({ ok: true, chat });
+ });
+}
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256
index f3345271..18fbd384 100644
--- a/apps/server/src/computer-use/shared/PARITY.sha256
+++ b/apps/server/src/computer-use/shared/PARITY.sha256
@@ -6,8 +6,8 @@ c601c77fd65561657b103b6ec3c5e4e822d38d4cf9d535da368cc2466698a9d5 policy.mjs
103c58c3908bebab415e5f1878abf752a2b07498f502855298a72cb58225798f policy.d.mts
17e55894695db8183a659a6c868bd8b1ac9348ccb0e0618b3d8aa94adf98d61a image.mjs
f11dd35d6db769398222387695262aeeac5dbaf9df16191d62a8811e8613e25d image.d.mts
-eea23441a7360c186044072fc0261a5b524e20dd639d05161dd968dc4bfae774 media.mjs
-25f747b53f5a1dbb5b2439b65bc728c508eecb62fbe2650904beffa8d3854b9c media.d.mts
+06f9604aa2d36d8cda25098e26088832e8b20cf9b0d756c5d745517c935684c9 media.mjs
+c12540410a558f4b48960c75b7a09e0660d6de679a6e49e2542427530fd9a3d4 media.d.mts
02be6c62de13f3c8a55a0167c751742086c16bfadb201e9ce8e7f1b955d2376d native-worker.mjs
-b5c9d77ac33f81a5cdc4688eae83310c0ec290d6f37db3fae3cb42760ea43db5 portable-host.mjs
-17c765f94dc3df46852db7bbe86c5682742c354dd8778f0499c05996f7b53865 portable-host.d.mts
+e209a9b5f4c87f035451cb8f0d8a98075c04cc85fcb1bc273a76c84c042a8395 portable-host.mjs
+97bff0780540617933f44779ea52d0131190eeb8b4d914cd4abbc59abcd87343 portable-host.d.mts
diff --git a/apps/server/src/computer-use/shared/media.d.mts b/apps/server/src/computer-use/shared/media.d.mts
index 583ceb1c..1566c9a5 100644
--- a/apps/server/src/computer-use/shared/media.d.mts
+++ b/apps/server/src/computer-use/shared/media.d.mts
@@ -45,6 +45,8 @@ export type MediaSpool = {
fullPath(sha: string): string;
thumb(sha: string): Buffer | null;
has(sha: string, ext: string): boolean;
+ ref(sha: string, chat: string): void;
+ release(sha: string, chat: string): boolean;
remove(sha: string): void;
};
export type MediaIndexState = { v: 1; seen: Record<string, { m: 0 | 1; f: 0 | 1 }>; model_bytes: number; full_bytes: number; frames: number };
@@ -64,7 +66,7 @@ export function parseImageDataUrl(url: unknown): { mime: string; bytes: Buffer }
export function thumbnailOf(bytes: Uint8Array, mime: string): Buffer | null;
export function createMediaSpool(dir: string): MediaSpool;
export function createMediaIndex(file: string): MediaIndex;
-export function rewriteMessageMedia<T>(message: T, ctx: { spool: MediaSpool; index: MediaIndex; limits?: { media_full_bytes?: number; media_model_bytes?: number } | null; thumbBudget?: number }): { message: T; changed: boolean; files: MediaFile[] };
+export function rewriteMessageMedia<T>(message: T, ctx: { spool: MediaSpool; index: MediaIndex; limits?: { media_full_bytes?: number; media_model_bytes?: number } | null; thumbBudget?: number; chat?: string }): { message: T; changed: boolean; files: MediaFile[] };
export function mediaRefsOf(message: unknown): Array<{ sha: string; ext: string }>;
export function fileBytes(file: string): number;
-export function pruneImages(messages: unknown[], options?: { keep?: number; drop?: number }): number;
+export function pruneImages(messages: unknown[], options?: { keep?: number; drop?: number; match?: (part: Record<string, unknown>) => boolean }): number;
diff --git a/apps/server/src/computer-use/shared/media.mjs b/apps/server/src/computer-use/shared/media.mjs
index 4675f03b..3f652c81 100644
--- a/apps/server/src/computer-use/shared/media.mjs
+++ b/apps/server/src/computer-use/shared/media.mjs
@@ -137,7 +137,32 @@ export function createMediaSpool(dir) {
fullPath(sha) { return path.join(dir, "full", `${sha}.png`); },
thumb(sha) { try { return readFileSync(path.join(dir, "thumb", `${sha}.jpg`)); } catch { return null; } },
has(sha, ext) { return existsSync(path.join(dir, `${sha}.${ext}`)); },
- /** Deletes a frame's local files (after its upload). */
+ /** Records that a chat references a frame (one spool serves every chat). */
+ ref(sha, chat) {
+ if (!SHA.test(String(sha)) || !chat) return;
+ const file = path.join(dir, "refs", `${sha}.json`);
+ const value = readJson(file);
+ const chats = new Set(Array.isArray(value?.chats) ? value.chats : []);
+ if (chats.has(chat)) return;
+ chats.add(chat);
+ writeAtomic(file, JSON.stringify({ chats: [...chats] }));
+ },
+ /**
+ * A chat no longer needs a frame (its archive holds it): the local files
+ * go when no other chat references it. Whether they were deleted.
+ */
+ release(sha, chat) {
+ if (!SHA.test(String(sha))) return false;
+ const file = path.join(dir, "refs", `${sha}.json`);
+ const value = readJson(file);
+ const chats = new Set(Array.isArray(value?.chats) ? value.chats : []);
+ chats.delete(chat);
+ if (chats.size) { writeAtomic(file, JSON.stringify({ chats: [...chats] })); return false; }
+ spool.remove(sha);
+ rmSync(file, { force: true });
+ return true;
+ },
+ /** Deletes a frame's local files whatever references it (prefer release). */
remove(sha) {
if (!SHA.test(String(sha))) return;
const meta = spool.meta(sha);
@@ -192,7 +217,8 @@ function budgets(limits) {
/**
* Replaces image data URLs in the tool parts of one engine message (in place
* on a structured copy; the input is not changed) with media references.
- * `ctx`: { spool, index, limits, thumbBudget }. Returns
+ * `ctx`: { spool, index, limits, thumbBudget, chat } (chat: the chat's id, for
+ * the spool's references). Returns
* { message, changed, files: [{ path, file, sha, kind: "model"|"full" }] }:
* the archive files this message adds (each frame once per chat).
* User file parts are left alone (the archive keeps those as attachments).
@@ -221,6 +247,7 @@ export function rewriteMessageMedia(message, ctx) {
let entry = ctx.index.entry(sha);
const first = !entry;
if (first) {
+ if (ctx.chat) ctx.spool.ref(sha, ctx.chat);
const model = ctx.index.state.model_bytes + meta.bytes <= budget.model;
const full = Boolean(meta.full) && model && ctx.index.state.full_bytes + meta.full.bytes <= budget.full && existsSync(ctx.spool.fullPath(sha));
entry = ctx.index.add(sha, { model, full, modelBytes: meta.bytes, fullBytes: meta.full?.bytes ?? 0 });
@@ -274,13 +301,14 @@ export const DROP_IMAGES = 8;
* keeps up to KEEP_IMAGES tool-result images; when the count passes it, the
* oldest DROP_IMAGES at a time are replaced by a text note, so the cached
* prefix stays stable for several steps. Mutates `messages` in place and
- * returns the number of images dropped. `messages`: engine WithParts list.
+ * returns the number of images dropped. `messages`: engine WithParts list;
+ * `match(part)` limits it to some tool parts (default: every tool part).
*/
-export function pruneImages(messages, { keep = KEEP_IMAGES, drop = DROP_IMAGES } = {}) {
+export function pruneImages(messages, { keep = KEEP_IMAGES, drop = DROP_IMAGES, match = () => true } = {}) {
const found = [];
for (const message of Array.isArray(messages) ? messages : []) {
for (const part of isRecord(message) && Array.isArray(message.parts) ? message.parts : []) {
- if (!isRecord(part) || part.type !== "tool" || !isRecord(part.state) || !Array.isArray(part.state.attachments)) continue;
+ if (!isRecord(part) || part.type !== "tool" || !isRecord(part.state) || !Array.isArray(part.state.attachments) || !match(part)) continue;
part.state.attachments.forEach((a, i) => { if (isRecord(a) && typeof a.mime === "string" && a.mime.startsWith("image/")) found.push({ part, index: i, attachment: a }); });
}
}
diff --git a/apps/server/src/computer-use/shared/portable-host.d.mts b/apps/server/src/computer-use/shared/portable-host.d.mts
index 48a6204b..1220b679 100644
--- a/apps/server/src/computer-use/shared/portable-host.d.mts
+++ b/apps/server/src/computer-use/shared/portable-host.d.mts
@@ -64,6 +64,10 @@ export type HostSessionState = {
log: Array<{ at: string; action: string; intent: string; ok: boolean; code: string | null }>;
frame: { sha: string; width: number; height: number; mime: string; data_url: string; at: string } | null;
chat: string | null;
+ /** The person hid the floating pill. */
+ hidden: boolean;
+ /** The window's native bounds, for placing the floating pill. */
+ window_bounds: { x: number; y: number; width: number; height: number };
};
export type McpConnection = { id: string; handle(message: unknown): Promise<Record<string, unknown> | null>; close(): Promise<void> };
export type PortableHost = {
diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjs
index c6dea5c1..14815059 100644
--- a/apps/server/src/computer-use/shared/portable-host.mjs
+++ b/apps/server/src/computer-use/shared/portable-host.mjs
@@ -40,6 +40,8 @@ import { actApproval, deniedApp } from "./policy.mjs";
const RPC = new Set(["initialize", "ping", "tools/list", "tools/call", "notifications/initialized", "notifications/cancelled"]);
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+// For timeouts raced against an answer: never keeps a short-lived process alive.
+const timeout = (ms) => new Promise((resolve) => { const timer = setTimeout(resolve, ms); timer.unref?.(); });
const now = () => new Date().toISOString();
const record = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
@@ -154,7 +156,7 @@ export async function createPortableHost({ native, spool = null, settings, conte
window: { id: String(s.window.id), title: s.window.title }, windows: (s.windows ?? []).map((w) => ({ id: String(w.id), title: w.title })),
approval: s.approval ?? null, actions: s.actions, actions_left: Math.max(0, limits().actions_per_session - s.actions),
seconds_left: Math.max(0, Math.ceil((s.expiresAt - Date.now()) / 1000)), can_continue: s.phase === "paused", log: s.log.slice(-30),
- frame: s.lastFrame ?? null, chat: s.chat ?? null,
+ frame: s.lastFrame ?? null, chat: s.chat ?? null, hidden: s.hidden === true, window_bounds: { ...s.window.bounds },
}));
const update = () => { try { onState(publicState()); } catch { /* UI is best effort */ } };
@@ -226,7 +228,7 @@ export async function createPortableHost({ native, spool = null, settings, conte
const png = encodePng(big, { level: 3 });
full = { bytes: png, width: big.width, height: big.height, sha256: sha256(png) };
}
- const ocr = "none";
+ const ocr = raw.ocr === "client" ? "client" : "none";
if (store && spool) {
try { spool.put({ bytes, mime, width: model.width, height: model.height, full, thumb: encodeJpeg(fitLongSide(model, 320), { quality: 70 }), ocr, masked }); } catch { /* the archive copy is best effort */ }
}
@@ -315,6 +317,8 @@ export async function createPortableHost({ native, spool = null, settings, conte
// Tolerant stale check: the window must still look like the observation (caret masked).
if (action.type !== "wait") {
const fresh = await native.call("capture", { window: current }, 15_000);
+ // The observation's grid was taken after masking: mask the fresh frame the same way.
+ paintBoxes(fresh, fresh.secure ?? []);
const grid = luminanceGrid(fitLongSide(fresh, l.model_long_side));
const compare = compareGrids(observed.grid, grid, { mask: observed.caret ? [observed.caret] : [] });
if (compare.stale) return finish({ ...common, approval, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });
@@ -350,12 +354,12 @@ export async function createPortableHost({ native, spool = null, settings, conte
async function askAct(s, request) {
const l = limits();
- const timeout = sleep(l.approval_timeout_seconds * 1000).then(() => ({ allow: false }));
+ const expired = timeout(l.approval_timeout_seconds * 1000).then(() => ({ allow: false }));
s.waiting = true; update();
try {
return await Promise.race([
Promise.resolve(approve.act({ session_id: s.id, request_id: randomUUID(), app: { app_id: s.window.appId, app_name: s.window.appName, bundle_or_exe: s.window.executable, pid: s.window.pid }, window_title: s.window.title, purpose: s.purpose, intent: request.intent, action: request.action, reasons: request.reasons, points: request.points, frame: { width: request.frame.model.width, height: request.frame.model.height, sha256: request.frame.sha, data_url: `data:${request.frame.mime};base64,${request.frame.bytes.toString("base64")}` } })).then((v) => ({ allow: v?.allow === true })),
- timeout,
+ expired,
]);
} catch {
return { allow: false };
@@ -407,7 +411,7 @@ export async function createPortableHost({ native, spool = null, settings, conte
answer = await Promise.race([
Promise.resolve(approve.open(request)).catch(() => ({ decision: "deny" })),
pendingAnswer,
- sleep(l.approval_timeout_seconds * 1000).then(() => ({ decision: "deny", timeout: true })),
+ timeout(l.approval_timeout_seconds * 1000).then(() => ({ decision: "deny", timeout: true })),
]);
s.pendingApproval = null;
}
diff --git a/apps/server/src/managed-policy-rules.ts b/apps/server/src/managed-policy-rules.ts
index f363b44f..e3c8bcce 100644
--- a/apps/server/src/managed-policy-rules.ts
+++ b/apps/server/src/managed-policy-rules.ts
@@ -2,6 +2,7 @@ import type { DesktopConfig, DesktopExecutionPolicy, DesktopPolicyKey } from "@o
import { z } from "zod";
import type { ApprovalMode } from "./approval-mode.js";
import { gitWorkflowPermissionRules } from "./git-command-policy.js";
+import { SCREEN_BASH_RULES } from "./computer-use/shared/policy.mjs";
import permissionModes from "./permissions/permission-modes.json" with { type: "json" };
export const managedPolicyActionSchema = z.enum([
@@ -107,6 +108,11 @@ export function legacyExecutionPermissions(
delete bash[pattern];
bash[pattern] = action;
}
+ // Commands that capture the screen or inject input ask (computer-use/shared/policy.mjs, the CLI's list).
+ for (const [pattern, action] of Object.entries(SCREEN_BASH_RULES)) {
+ delete bash[pattern];
+ bash[pattern] = action;
+ }
permissions = { bash, read: normal.read as Record<string, PermissionAction> };
}
for (const rule of executionRules(policy)) {
diff --git a/apps/server/src/mcp-allow/state.ts b/apps/server/src/mcp-allow/state.ts
index 3dbf8bc3..36bc6103 100644
--- a/apps/server/src/mcp-allow/state.ts
+++ b/apps/server/src/mcp-allow/state.ts
@@ -6,6 +6,7 @@
//
// The engine gets the three paths in its environment; its plugin
// (opencode-plugins/omnirush-mcp-policy.ts) reads them at each project's start.
+import { computerUseEngineEnv } from "../computer-use/desktop.js";
import { mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
import path from "node:path";
@@ -53,6 +54,8 @@ export function mcpEngineEnv(stateDir: string, env: Record<string, string | unde
// The engine starts no MCP server unless the MCP plugin marked its config: a missing or broken plugin
// fails closed (engine 1.18.32-ux4 and later).
OMNIRUSH_MCP_FILTER: "required",
+ // The built-in computer use server (computer-use/desktop.ts), read by the same plugin.
+ ...computerUseEngineEnv(stateDir),
};
}
diff --git a/apps/server/src/omnirush-gateway-broker.ts b/apps/server/src/omnirush-gateway-broker.ts
index bc433d67..265bccd9 100644
--- a/apps/server/src/omnirush-gateway-broker.ts
+++ b/apps/server/src/omnirush-gateway-broker.ts
@@ -1684,6 +1684,8 @@ export class OmniRushGatewayBroker {
: {}),
// The model's own web search (web-search.ts reads it; absent is off).
...(isRecord(payload.web_search) ? { web_search: payload.web_search } : {}),
+ // Computer use (computer-use/desktop.ts reads it; absent is off).
+ ...(isRecord(payload.computer_use) ? { computer_use: payload.computer_use } : {}),
};
} catch {
return null;
diff --git a/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts b/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts
index 0ac906b0..e60a0bef 100644
--- a/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts
+++ b/apps/server/src/opencode-plugins/omnirush-mcp-policy.ts
@@ -7,14 +7,22 @@
// keyed by their id, pinned, paths checked, and run in the project's sandbox
// when it is on. The report (OMNIRUSH_MCP_REPORT_DIR) says which and why.
// - tool.execute.before: servers limited to local pages refuse other pages.
+// - computer use (computer-use/desktop.ts): the app's built-in `computer-use`
+// server is added when it is on (its id is always reserved), sub-agents get
+// no computer tools, and old screenshots of computer tools are pruned from
+// what the model is sent (the stored chat is untouched).
//
// No policy file (OMNIRUSH_MCP_POLICY_FILE missing or unreadable): no MCP.
// The engine calls every function a plugin module exports: export only the plugin.
-import { existsSync } from "node:fs";
+import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import path from "node:path";
-import { createMcpPolicyHooks, type McpBox } from "../mcp-allow/mcp-allow.mjs";
+import { createMcpPolicyHooks, reportPath, type McpBox } from "../mcp-allow/mcp-allow.mjs";
import { readWorkspaceMcp } from "../mcp-allow/state.js";
+import { computerUseBuiltins, COMPUTER_USE_DIR_ENV } from "../computer-use/desktop.js";
+import { isComputerTool, SERVER_ID } from "../computer-use/shared/contract.mjs";
+import { pruneImages } from "../computer-use/shared/media.mjs";
+import { subagentPermissionRules } from "../computer-use/shared/policy.mjs";
import { boxHome, desktopProjects, keyOf, record } from "../box/plugin-lib.js";
function isRecord(value: unknown): value is Record<string, unknown> {
@@ -38,6 +46,37 @@ function boxFor(root: string): McpBox | null {
return { shell, key, croot: typeof rec.croot === "string" ? rec.croot : null };
}
+const SUBAGENTS = ["general", "explore"];
+
+/** Adds the left-out `computer-use` entry to the project's MCP report (Settings > MCP). */
+function reportReserved(root: string): void {
+ const dir = process.env.OMNIRUSH_MCP_REPORT_DIR;
+ if (!dir) return;
+ try {
+ const file = reportPath(dir, root);
+ const report: unknown = JSON.parse(readFileSync(file, "utf8"));
+ if (!isRecord(report) || !Array.isArray(report.decisions)) return;
+ report.decisions = [...report.decisions.filter((d) => !(isRecord(d) && d.name === SERVER_ID)), { name: SERVER_ID, status: "blocked", code: "reserved", reason: "This name belongs to the app's built-in computer use; this entry is left out." }];
+ const temp = `${file}.${process.pid}.cu.tmp`;
+ writeFileSync(temp, JSON.stringify(report), { mode: 0o600 });
+ renameSync(temp, file);
+ } catch {
+ // The report is for display only.
+ }
+}
+
+/** No computer tools in sub-agents (one host lease per app; v1). */
+function withoutSubagentComputerTools(target: Record<string, unknown>): void {
+ const agents = isRecord(target.agent) ? { ...target.agent } : {};
+ const names = new Set([...SUBAGENTS, ...Object.entries(agents).filter(([, value]) => isRecord(value) && value.mode === "subagent").map(([name]) => name)]);
+ for (const name of names) {
+ const agent = isRecord(agents[name]) ? { ...agents[name] } : {};
+ const permission = typeof agent.permission === "string" ? { "*": agent.permission } : isRecord(agent.permission) ? { ...agent.permission } : {};
+ agents[name] = { ...agent, permission: { ...permission, ...subagentPermissionRules() } };
+ }
+ target.agent = agents;
+}
+
export default async function omnirushMcpPolicy(input?: { directory?: string }) {
const root = path.resolve(input?.directory || process.cwd());
const hooks = createMcpPolicyHooks({
@@ -57,8 +96,29 @@ export default async function omnirushMcpPolicy(input?: { directory?: string })
// The app-added servers are left out; the config's own still go through the list.
}
}
+ // The id `computer-use` belongs to the app's built-in server: a config entry with it is left out (and reported).
+ const reserved = target && isRecord(target.mcp) && Object.hasOwn(target.mcp, SERVER_ID);
+ if (target && reserved && isRecord(target.mcp)) {
+ const { [SERVER_ID]: _dropped, ...rest } = target.mcp;
+ target.mcp = rest;
+ }
// Fails closed: an error while applying the list leaves no servers at all.
await hooks.config(configInput, output);
+ if (reserved) reportReserved(root);
+ const builtin = target ? computerUseBuiltins(process.env[COMPUTER_USE_DIR_ENV])[SERVER_ID] : null;
+ if (target && builtin) {
+ target.mcp = { ...(isRecord(target.mcp) ? target.mcp : {}), [SERVER_ID]: builtin };
+ // The host asks the person itself (never the engine's permission prompts), in every mode.
+ const permission: Record<string, unknown> = typeof target.permission === "string" ? { "*": target.permission } : isRecord(target.permission) ? { ...target.permission } : {};
+ delete permission[`${SERVER_ID}_*`];
+ target.permission = { ...permission, [`${SERVER_ID}_*`]: "allow" };
+ withoutSubagentComputerTools(target);
+ }
+ },
+ // Old screenshots of computer tools leave the model's context in blocks (keep 12, drop 8);
+ // compaction requests serialize messages to text and carry no images anyway.
+ "experimental.chat.messages.transform": async (_input: unknown, output: { messages?: unknown[] }) => {
+ if (Array.isArray(output?.messages)) pruneImages(output.messages, { match: (part: Record<string, unknown>) => isComputerTool(part.tool) });
},
"tool.execute.before": hooks["tool.execute.before"],
};
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index 43f4751d..fd22c442 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -183,6 +183,8 @@ import { buildathonProjectsFile, reproStateFromBody, writeReproState } from "./b
import { prepareBoxInstallLog } from "./box/install-log.js";
import { rememberServerDefault, watchServerDefault } from "./box/launch.js";
import { rememberWebSearchSwitch, watchWebSearchSwitch } from "./web-search.js";
+import { createChatTracker, rememberComputerUseSwitch } from "./computer-use/desktop.js";
+import { registerComputerUseRoutes } from "./computer-use/routes.js";
import { normalizeBackend } from "./box/backends/index.js";
import { buildOmniRushRuntimeConfigObject, omnirushGatewayConfigured, omnirushRuntimeConfigFilePath, writeOmniRushRuntimeConfigFile } from "./omnirush-runtime-config.js";
import { assertMcpAllowed, mcpAllowed, mcpDisabledMessage, mcpPolicyForConfig, refuseAccountLinkedMcp } from "./mcp-policy.js";
@@ -281,6 +283,26 @@ const AUTO_COMPACT_SEED_WAIT_MS = 5_000;
/** How long a sign-out waits for queued uploads to go (POST /omnirush/capture/flush); the rest stays queued for the account. */
const SIGN_OUT_FLUSH_MS = 10_000;
const turnActivityByServer = new WeakMap<ServerConfig, TurnActivity>();
+/** Fresh capabilities for computer use (Electron main asks every 60 s while a session is open), per server. */
+const computerUseCapabilitiesByServer = new WeakMap<ServerConfig, (maxAgeMs: number) => Promise<unknown>>();
+/** The chats prompted most recently, per server: a computer use call belongs to the latest one still running. */
+const computerUseChatsByServer = new WeakMap<ServerConfig, ReturnType<typeof createChatTracker>>();
+function computerUseChats(config: ServerConfig): ReturnType<typeof createChatTracker> {
+ let chats = computerUseChatsByServer.get(config);
+ if (!chats) {
+ chats = createChatTracker();
+ computerUseChatsByServer.set(config, chats);
+ }
+ return chats;
+}
+
+/** Restarts the engine's projects (when idle) so they pick up the built-in computer use server. */
+async function applyComputerUseEngine(config: ServerConfig): Promise<void> {
+ const pool = enginePoolForConfig(config);
+ const workspaces = config.workspaces.filter((workspace) => resolveWorkspaceOpencodeConnection(config, workspace).baseUrl?.trim());
+ const workspace = findManagedEngineWorkspace(workspaces) ?? workspaces[0];
+ if (pool && workspace) await pool.requestRollover({ reason: "computer_use", workspace, manual: true });
+}
/** Whether the agent works on this server's engine (turn-activity.ts), for an embedding app that asked (onTurnActivity). */
function turnActivityFor(config: ServerConfig): TurnActivity | null {
@@ -1266,6 +1288,7 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {
}));
}
if (gatewayBroker.enabled) sessionIntegrityByServer.set(config, (sessionId, options) => gatewayBroker.sessionIntegrity(sessionId, options));
+ if (gatewayBroker.enabled) computerUseCapabilitiesByServer.set(config, (maxAgeMs) => gatewayBroker.sessionUploadCapabilities({ strict: true, maxAgeMs }));
voiceServicesByServer.set(config, new OmniRushVoiceService(gatewayBroker, {
log: (level, message, attributes) => logger.log(level, message, attributes),
}));
@@ -1332,6 +1355,7 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {
let stopBoxDefaultWatch: () => void = () => {};
let stopWebSearchWatch: () => void = () => {};
let stopMcpPolicyWatch: () => void = () => {};
+ let stopComputerUseWatch: () => void = () => {};
if (gatewayBroker.enabled) {
// Read before the first plan and the engine's first config (at most 1.5 s, else the last answer is
// used until it comes): a switch flipped or rolled back on the server applies at this start.
@@ -1354,6 +1378,15 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {
});
// Read again while the app stays open (from the capabilities cache: at most one request per its
// TTL), so a switch flipped on the server reaches an app started before it without a restart.
+ // Computer use (computer-use/desktop.ts): remembered from the same answer; a change restarts the
+ // engine's projects so the built-in server comes or goes. Electron main reads it again every 60 s
+ // while a computer use session is open (POST /computer-use/refresh).
+ void gatewayBroker.sessionUploadCapabilities({ strict: true }).then((capabilities) => (rememberComputerUseSwitch(runtimeStorageDir(config), capabilities) ? applyComputerUseEngine(config) : undefined)).catch(() => undefined);
+ const computerUseTimer = setInterval(() => {
+ void gatewayBroker.sessionUploadCapabilities({ strict: true }).then((capabilities) => (rememberComputerUseSwitch(runtimeStorageDir(config), capabilities) ? applyComputerUseEngine(config) : undefined)).catch(() => undefined);
+ }, BOX_DEFAULT_REFRESH_MS);
+ computerUseTimer.unref?.();
+ stopComputerUseWatch = () => clearInterval(computerUseTimer);
stopMcpPolicyWatch = watchMcpPolicy({
stateDir: runtimeStorageDir(config),
read: () => gatewayBroker.sessionUploadCapabilities(),
@@ -1731,6 +1764,7 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {
stopBoxDefaultWatch();
stopWebSearchWatch();
stopMcpPolicyWatch();
+ stopComputerUseWatch();
bestPracticesReloads.stop();
// First, and synchronously: archive part uploads in flight are aborted now, not after the other shutdown steps.
const captureStopped = capture.stop();
@@ -1823,6 +1857,7 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {
stopBoxDefaultWatch();
stopWebSearchWatch();
stopMcpPolicyWatch();
+ stopComputerUseWatch();
const archiveFinals = accountStillStored();
if (boxes.hasChats() && await Promise.race([archiveFinals.catch(() => false), new Promise<false>((resolvePromise) => setTimeout(() => resolvePromise(false), 1_000).unref?.())])) {
await boxes.finals({ budgetMs: 10_000 }).catch(() => []);
@@ -1926,6 +1961,7 @@ export async function proxyOpencodeV2Request(input: {
// The agent works from here (the prompt is on its way) until the engine runs no session.
const turnActivity = promptDispatch && collectible && sessionId !== null ? turnActivityFor(input.config) : null;
turnActivity?.promptDispatched();
+ if (promptDispatch && sessionId !== null) computerUseChats(input.config).mark(sessionId);
const target = new URL(input.connection.url);
target.pathname = forwardedPath;
target.search = input.url.search;
@@ -4838,6 +4874,24 @@ function createRoutes(
scopeRank,
});
+ // Computer use (computer-use/routes.ts): for Electron main, which runs the host.
+ registerComputerUseRoutes({
+ routes,
+ stateDir: () => runtimeStorageDir(config),
+ jsonResponse,
+ readJsonBody,
+ approvalMode: () => capturePermissionMode(config),
+ readCapabilities: async (maxAgeMs) => {
+ const read = computerUseCapabilitiesByServer.get(config);
+ if (!read) throw new Error("capabilities unavailable");
+ return read(maxAgeMs);
+ },
+ applyEngine: () => applyComputerUseEngine(config),
+ chats: computerUseChats(config),
+ runningSessions: async () => null,
+ recordTrace: (sessionId, type, data) => recordOpenSessionTrace(config, sessionId, type, data),
+ });
+
// The desktop editor. A save or a hunk review that names a chat the session
// uploader is tracking adds a "user_edit" / "user_review" trace event to it;
// nothing else in a chat's upload changes (editor-events.ts).
diff --git a/apps/server/src/session-archive/index.ts b/apps/server/src/session-archive/index.ts
index 82b4f92a..db8a20c7 100644
--- a/apps/server/src/session-archive/index.ts
+++ b/apps/server/src/session-archive/index.ts
@@ -64,6 +64,7 @@ import { SEAL_CONTENT } from "./seal.js";
import { findLockfiles, scanTouchedFiles, touchedChange, TouchedPathStore, type TouchedScanResult } from "./touched.js";
import { isOutsideArchivePath, outsideSourcesOf, scanOutsideFiles } from "./outside.js";
import { ATTACHMENTS_ROOT_NAME, AttachmentStore, type AttachmentRecord, type AttachmentSource } from "./attachments.js";
+import { MEDIA_ROOT_NAME, MediaStore, type MediaRecord } from "./media-store.js";
import {
ARCHIVE_SCHEMA_V2,
captureV2Override,
@@ -517,7 +518,7 @@ function repoMarkers(entries: readonly ArchiveEntry[]): Array<{ path: string; gi
if (entry.type === "symlink") continue;
const parts = entry.path.split("/");
if (parts[parts.length - 1] !== ".git" || parts.slice(0, -1).includes(".git")) continue;
- if (parts[0] === ATTACHMENTS_ROOT_NAME || parts[0] === ENCLOSING_REPO_ROOT_NAME || isOutsideArchivePath(entry.path)) continue;
+ if (parts[0] === ATTACHMENTS_ROOT_NAME || parts[0] === MEDIA_ROOT_NAME || parts[0] === ENCLOSING_REPO_ROOT_NAME || isOutsideArchivePath(entry.path)) continue;
markers.push({ path: entry.path, gitfile: entry.type === "file" });
}
return markers;
@@ -568,6 +569,7 @@ export class SessionArchiver {
private readonly touched: TouchedPathStore;
/** Capture v2: staged prompt attachments (attachments.ts). */
private readonly attachments: AttachmentStore;
+ private readonly media: MediaStore;
/** Capture v2: start scans waiting for their session's base. */
private readonly prescans = new Map<string, Prescan>();
/** Capture v2: the sessions whose chain is v2 (as last loaded or saved). */
@@ -626,6 +628,7 @@ export class SessionArchiver {
this.includeCredentials = options.archiveIncludeCredentialFiles === true;
this.filesUsed = new FilesUsedStore(join(this.dir, "files-used"));
this.filesUsedTracker = new FilesUsedTracker({ archive: this, appDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, log: this.log, ...(options.filesUsedSandbox ? { sandboxFor: options.filesUsedSandbox } : {}) });
+ this.media = new MediaStore(join(this.dir, "media"));
this.attachments = new AttachmentStore(join(this.dir, "attachments"), (path) => !this.includeCredentials && isUploadPathDenied(path), { appDirs: this.appDirs, includeCredentialFiles: this.includeCredentials });
this.touched = new TouchedPathStore(this.dirs.touched, {
ready: () => this.start(),
@@ -1049,6 +1052,19 @@ export class SessionArchiver {
return this.attachments.add(sessionId, state.root || root, sources);
}
+ /**
+ * Screenshots and other tool-result images of the chat (media-store.ts),
+ * staged now and archived by the session's next capture under `__media__/`.
+ * Never throws.
+ */
+ async recordMedia(sessionId: string, records: readonly MediaRecord[]): Promise<number> {
+ if (records.length === 0 || this.disabled || !SESSION_ID_PATTERN.test(sessionId) || this.stoppedSessions.has(sessionId)) return 0;
+ await this.start();
+ const state = await this.loadSession(sessionId);
+ if (!state?.v2 || state.stopped) return 0;
+ return this.media.add(sessionId, records);
+ }
+
/**
* Capture v2: a binary file of the workspace the session uploader found
* (never sent as text). A v2 chain archives it byte for byte, also a
@@ -1621,12 +1637,13 @@ export class SessionArchiver {
const lockfiles = await findLockfiles(state.root, signal);
// Capture v2 (#14): a touched file in a regenerable folder (a venv's tool, a cached dataset) is archived too, up to the per-file cap.
const paths = new Set([...touchedPaths.filter((path) => (v2 ? !path.split("/").slice(0, -1).includes(".git") : !inRegenerableDir(path))), ...lockfiles.filter((path) => !inRegenerableDir(path))]);
- for (const entry of baseline) if (!isOutsideArchivePath(entry.path) && !entry.path.startsWith(`${ATTACHMENTS_ROOT_NAME}/`)) paths.add(entry.path);
+ for (const entry of baseline) if (!isOutsideArchivePath(entry.path) && !entry.path.startsWith(`${ATTACHMENTS_ROOT_NAME}/`) && !entry.path.startsWith(`${MEDIA_ROOT_NAME}/`)) paths.add(entry.path);
const scan = await scanTouchedFiles(state.root, paths, { excludedDirs: this.appDirs, includeCredentialFiles: this.includeCredentials, includeIgnored: true, hashCache: cache, ...(signal ? { signal } : {}) });
if (excludedList) this.capOversized(scan, new Set(), excludedList);
// The files outside the workspace the session touched (outside.ts).
mergeScans(scan, await this.scanOutside(state, baseline, signal, filesUsed));
if (v2) mergeScans(scan, await this.attachmentEntries(sessionId));
+ if (v2) mergeScans(scan, await this.mediaEntries(sessionId));
mergeScans(scan, await this.sandboxEntries(sessionId));
if (filesUsed) mergeStaged(scan, await this.filesUsed.entries(sessionId));
if (options.trigger === "pre_tool" && options.before !== undefined) scanAsOf(scan.entries, new Map(baseline.map((entry) => [entry.path, entry])), options.before);
@@ -1658,6 +1675,7 @@ export class SessionArchiver {
// The files outside the workspace the session touched (outside.ts).
mergeScans(scan, await this.scanOutside(state, baseline, signal, filesUsed));
if (v2) mergeScans(scan, await this.attachmentEntries(sessionId));
+ if (v2) mergeScans(scan, await this.mediaEntries(sessionId));
mergeScans(scan, await this.sandboxEntries(sessionId));
if (filesUsed) mergeStaged(scan, await this.filesUsed.entries(sessionId));
// Capture v2: the .git of the repository above a session folder (enclosing.ts).
@@ -1690,6 +1708,8 @@ export class SessionArchiver {
const archiveId = this.random.uuid();
const sequence = state.next_sequence;
const createdAt = this.now();
+ // The chat's screenshots in this archive: counted by the server apart from the project's files.
+ const mediaBytes = files.reduce((sum, entry) => (entry.type === "file" && entry.path.startsWith(`${MEDIA_ROOT_NAME}/`) ? sum + entry.size : sum), 0);
let extraMembers: Array<{ name: string; content: Buffer }> = [];
if (v2 && excludedList) {
const stateKind: StateKind = kind === "base" ? "start" : options.trigger === "pre_tool" ? "pre_tool" : options.trigger === "final" ? "final" : "after";
@@ -1802,6 +1822,7 @@ export class SessionArchiver {
kid: sealed.kid,
content: SEAL_CONTENT,
marker: state.marker,
+ ...(mediaBytes > 0 ? { media_bytes: mediaBytes } : {}),
};
const record: QueueRecord = {
v: 1,
@@ -2052,6 +2073,21 @@ export class SessionArchiver {
return { entries, gone: new Set(), excluded: emptyExcludedCounts(), ignored: 0 };
}
+ private async mediaEntries(sessionId: string): Promise<TouchedScanResult> {
+ const entries: ScannedEntry[] = [];
+ for (const record of await this.media.present(sessionId)) {
+ try {
+ const stats = await lstat(record.file, { bigint: true });
+ if (!stats.isFile()) continue;
+ entries.push({ path: record.path, type: "file", size: Number(stats.size), sha256: await sha256Path(record.file), source: record.file, ...statFields(stats), mode: 0o644 });
+ } catch {
+ // Gone from the spool: not archived.
+ }
+ }
+ entries.sort((left, right) => compareArchivePaths(left.path, right.path));
+ return { entries, gone: new Set(), excluded: emptyExcludedCounts(), ignored: 0 };
+ }
+
/**
* The files outside the workspace the session touched, and those an
* earlier archive of the chain holds (to see them change or go), scanned
diff --git a/apps/server/src/session-archive/lifecycle.ts b/apps/server/src/session-archive/lifecycle.ts
index 13a8c458..a4d482af 100644
--- a/apps/server/src/session-archive/lifecycle.ts
+++ b/apps/server/src/session-archive/lifecycle.ts
@@ -11,6 +11,7 @@
* for what that missed, when the app starts again. Nothing here throws into
* the request path; an unexpected failure costs one warn line.
*/
+import type { MediaRecord } from "./media-store.js";
import { realpath } from "node:fs/promises";
import { attachmentsFromMessages } from "./attachments.js";
@@ -22,7 +23,7 @@ import type { UsedToolCall } from "./files-used.js";
export type ProjectArchiver = Pick<
SessionArchiver,
"captureBase" | "captureDelta" | "captureFinal" | "startFinalCandidates" | "recordTouched" | "forgetTouched" | "drain" | "signOut" | "stop"
-> & Partial<Pick<SessionArchiver, "claimAccount" | "discardAll" | "pendingStatus" | "startManifest" | "hasStartManifest" | "captureState" | "recordAttachments" | "recordBinary" | "filesUsedToolCallEnded" | "filesUsedTurnStarted" | "filesUsedTurnEnded" | "filesUsedMaybe" | "filesUsedStatus">>;
+> & Partial<Pick<SessionArchiver, "claimAccount" | "discardAll" | "pendingStatus" | "startManifest" | "hasStartManifest" | "captureState" | "recordAttachments" | "recordMedia" | "recordBinary" | "filesUsedToolCallEnded" | "filesUsedTurnStarted" | "filesUsedTurnEnded" | "filesUsedMaybe" | "filesUsedStatus">>;
/** Engine reads for one session, resolving to the parsed JSON, or null when it cannot be read. */
export type ArchiveEngineReads = {
@@ -398,6 +399,17 @@ export class ProjectArchiveLifecycle {
});
}
+ /** Screenshots of the chat's tool results (computer-use/media-capture.ts), staged for its next archive under `__media__/`. */
+ mediaFiles(sessionId: string, records: readonly MediaRecord[]): void {
+ const record = this.sessions.get(sessionId);
+ if (!this.active || this.consentOff || !record || !this.archiver.recordMedia) return;
+ const stage = this.archiver.recordMedia.bind(this.archiver);
+ this.schedule(sessionId, "media", async () => {
+ if (this.sessions.get(sessionId) !== record) return;
+ await stage(sessionId, records);
+ });
+ }
+
/**
* The session settled after a turn. `messages` are the engine's messages
* the observer read at that point; the delta carries their completed-turn
diff --git a/apps/server/src/session-archive/manifest.ts b/apps/server/src/session-archive/manifest.ts
index 8942aebd..cedf65d0 100644
--- a/apps/server/src/session-archive/manifest.ts
+++ b/apps/server/src/session-archive/manifest.ts
@@ -31,6 +31,8 @@ export { ATTACHMENTS_ROOT_NAME };
export const ENCLOSING_REPO_ROOT_NAME = "__enclosing_repo__";
/** The Linux sandbox's turn checkpoints (box/, index.ts sandboxEntries); a workspace entry of that name is reserved too. */
export const SANDBOX_ROOT_NAME = "__sandbox__";
+/** Screenshots and other tool-result images (media-store.ts); a workspace entry of that name is reserved too. */
+export const MEDIA_ROOT_NAME = "__media__";
/** A git config file larger than this is left out (`too_large`): it is never read whole to scrub it, and never archived raw. */
const MAX_SCRUBBED_CONFIG_BYTES = 1024 * 1024;
export const STAT_CONCURRENCY = 64;
@@ -708,7 +710,7 @@ export async function scanArchiveTree(root: string, options: ScanOptions = {}):
for (const bytes of raw) {
const name = decodeName(bytes);
if (name === null) excluded.non_utf8 += 1;
- else if (!dir.rel && (name === RESERVED_ROOT_NAME || name === OUTSIDE_ROOT_NAME || name === ATTACHMENTS_ROOT_NAME || name === ENCLOSING_REPO_ROOT_NAME || name === SANDBOX_ROOT_NAME)) {
+ else if (!dir.rel && (name === RESERVED_ROOT_NAME || name === OUTSIDE_ROOT_NAME || name === ATTACHMENTS_ROOT_NAME || name === ENCLOSING_REPO_ROOT_NAME || name === SANDBOX_ROOT_NAME || name === MEDIA_ROOT_NAME)) {
excluded.reserved += 1;
note(name, "reserved", "dir");
}
diff --git a/apps/server/src/session-archive/media-store.ts b/apps/server/src/session-archive/media-store.ts
new file mode 100644
index 00000000..f6fb64eb
--- /dev/null
+++ b/apps/server/src/session-archive/media-store.ts
@@ -0,0 +1,90 @@
+/**
+ * Screenshots and other tool-result images of a chat (computer-use/shared/media.mjs),
+ * staged for its project archive under `__media__/` (MEDIA_ROOT_NAME):
+ * __media__/<sha256>.<ext> the exact bytes the model saw
+ * __media__/full/<sha256>.png full resolution, only when larger
+ * The bytes stay in the local media spool until archived; every capture of
+ * the chain lists all of them again (like attachments), so none reads as deleted.
+ * Their caps are the media module's (media_model_bytes / media_full_bytes per
+ * chat), apart from the project's files.
+ */
+import { appendFile, lstat, mkdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { stateKey } from "./files.js";
+import { MEDIA_ROOT_NAME } from "./manifest.js";
+
+export { MEDIA_ROOT_NAME };
+
+export type MediaRecord = { path: string; file: string; sha: string; kind: "model" | "full"; bytes: number };
+
+const PATH = /^__media__\/(full\/)?[a-f0-9]{64}\.(png|jpg|gif|webp)$/;
+
+export class MediaStore {
+ constructor(private readonly dir: string) {}
+
+ private indexOf(sessionId: string): string {
+ return join(this.dir, `${stateKey(sessionId)}.jsonl`);
+ }
+
+ /** Stages frames for a chat (each archive path once). Never throws. */
+ async add(sessionId: string, records: readonly MediaRecord[]): Promise<number> {
+ const valid = records.filter((r) => PATH.test(r.path) && typeof r.file === "string" && r.file.length > 0 && Number.isInteger(r.bytes) && r.bytes > 0);
+ if (!valid.length) return 0;
+ try {
+ const known = new Set((await this.list(sessionId)).map((r) => r.path));
+ const fresh: MediaRecord[] = [];
+ for (const record of valid) {
+ if (known.has(record.path)) continue;
+ known.add(record.path);
+ fresh.push(record);
+ }
+ if (!fresh.length) return 0;
+ await mkdir(this.dir, { recursive: true, mode: 0o700 });
+ await appendFile(this.indexOf(sessionId), fresh.map((r) => `${JSON.stringify(r)}\n`).join(""), { mode: 0o600 });
+ return fresh.length;
+ } catch {
+ return 0;
+ }
+ }
+
+ async list(sessionId: string): Promise<MediaRecord[]> {
+ let text = "";
+ try {
+ text = await readFile(this.indexOf(sessionId), "utf8");
+ } catch {
+ return [];
+ }
+ const out: MediaRecord[] = [];
+ const seen = new Set<string>();
+ for (const line of text.split("\n")) {
+ if (!line.trim()) continue;
+ try {
+ const value: unknown = JSON.parse(line);
+ if (typeof value !== "object" || value === null) continue;
+ const record = value as Partial<MediaRecord>;
+ if (typeof record.path === "string" && PATH.test(record.path) && typeof record.file === "string" && typeof record.sha === "string" && !seen.has(record.path)) {
+ seen.add(record.path);
+ out.push({ path: record.path, file: record.file, sha: record.sha, kind: record.kind === "full" ? "full" : "model", bytes: Number(record.bytes) || 0 });
+ }
+ } catch {
+ // A torn last line: skipped.
+ }
+ }
+ return out;
+ }
+
+ /** The staged frames still in the spool, with their sizes now. */
+ async present(sessionId: string): Promise<Array<MediaRecord & { size: number }>> {
+ const out: Array<MediaRecord & { size: number }> = [];
+ for (const record of await this.list(sessionId)) {
+ try {
+ const stats = await lstat(record.file);
+ if (stats.isFile()) out.push({ ...record, size: stats.size });
+ } catch {
+ // Gone from the spool: not archived again.
+ }
+ }
+ return out;
+ }
+}
diff --git a/apps/server/src/session-archive/upload.ts b/apps/server/src/session-archive/upload.ts
index eb743c72..971df030 100644
--- a/apps/server/src/session-archive/upload.ts
+++ b/apps/server/src/session-archive/upload.ts
@@ -121,6 +121,8 @@ export type ArchiveCreateRequest = {
kid: string;
content: typeof SEAL_CONTENT;
marker: string;
+ /** Bytes of the `__media__/` files in this archive (media-store.ts): the server counts them apart from the project's. */
+ media_bytes?: number;
};
export type ArchiveUploadState = {
diff --git a/apps/server/src/session-upload-observer.ts b/apps/server/src/session-upload-observer.ts
index ac84f2e3..b346f9e9 100644
--- a/apps/server/src/session-upload-observer.ts
+++ b/apps/server/src/session-upload-observer.ts
@@ -619,6 +619,27 @@ export function slimEngineMessage(message: unknown, maxBytes: number | null = MA
/** A message read from the engine; `whole` is false for one known only by its info (too large to read). */
type EngineMessage = { message: unknown; whole: boolean };
+/**
+ * Applied to every message read from the engine, before any budget: image
+ * data URLs in tool results become media references (computer-use/media-capture.ts).
+ * The capture host sets it once; none: messages pass as they are.
+ */
+let engineMessageRewrite: ((sessionId: string, message: unknown) => unknown) | null = null;
+export function setEngineMessageRewrite(rewrite: ((sessionId: string, message: unknown) => unknown) | null): void {
+ engineMessageRewrite = rewrite;
+}
+
+function rewritten(path: string, message: unknown): unknown {
+ if (!engineMessageRewrite) return message;
+ const match = /\/session\/([^/]+)\/(?:message|context)$/.exec(path);
+ if (!match?.[1]) return message;
+ try {
+ return engineMessageRewrite(decodeURIComponent(match[1]), message);
+ } catch {
+ return message;
+ }
+}
+
/**
* A session's messages, newest first. A v1 engine is read a page at a
* time, each page parsed under the read cap: after a page over it, the
@@ -632,7 +653,7 @@ async function* engineMessages(fetchEngine: EngineFetch, path: string, paged: bo
const payload = await readEngineJson(fetchEngine, path);
const list = isRecord(payload) && "data" in payload ? payload.data : payload;
if (!Array.isArray(list)) throw new Error("the engine messages could not be read");
- for (const message of list.reverse()) yield { message: slimEngineMessage(message, null), whole: true };
+ for (const message of list.reverse()) yield { message: slimEngineMessage(rewritten(path, message), null), whole: true };
return;
}
let ceiling = MESSAGE_PAGE_SIZE;
@@ -651,7 +672,7 @@ async function* engineMessages(fetchEngine: EngineFetch, path: string, paged: bo
const alone = limit === 1;
const items = await readObservedResponse(response, alone ? MAX_SINGLE_MESSAGE_READ_BYTES : MAX_ENGINE_READ_BYTES);
if (!Array.isArray(items)) throw new Error("the engine messages could not be read");
- page = items.map((message) => ({ message: slimEngineMessage(message, alone ? MAX_TURN_MESSAGE_BYTES : null), whole: true }));
+ page = items.map((message) => ({ message: slimEngineMessage(rewritten(path, message), alone ? MAX_TURN_MESSAGE_BYTES : null), whole: true }));
} catch (error) {
if (!(error instanceof OversizedEngineResponse)) throw error;
if (limit > 1) {
diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.ts
index accc1f4a..660052b6 100644
--- a/apps/server/src/session-uploader.ts
+++ b/apps/server/src/session-uploader.ts
@@ -43,6 +43,7 @@ import bundledBestPractices from "./bundled-best-practices.json" with { type: "j
import serverPackage from "../package.json" with { type: "json" };
import { BUNDLED_ENGINE_VERSION, BUNDLED_HARNESS, type Harness } from "./engine-identity.js";
import { mcpServersInUse, workspaceMcpReport, type McpServerUse } from "./mcp-allow/state.js";
+import type { ComputerUseEnvironment } from "./computer-use/desktop.js";
const execFileAsync = promisify(execFile);
@@ -310,6 +311,8 @@ export type UploadEnvironment = {
sandbox?: BoxLabel;
/** The MCP servers the project's engine ran (mcp-allow/state.ts); absent when none. */
mcp_servers?: McpServerUse[];
+ /** Computer use in this chat (computer-use/desktop.ts): only when it was used. */
+ computer_use?: ComputerUseEnvironment;
};
/** A box chat's toolchain without this machine's `system` and `shell` blocks. */
@@ -874,6 +877,8 @@ export type TraceCapabilities = {
auto_compact?: { enabled: boolean; after_turns: number; min_context_tokens?: number };
/** The model's own web search (web-search.ts): the block as the server sent it; absent is off. */
web_search?: Record<string, unknown>;
+ /** Computer use (computer-use/desktop.ts): the block as the server sent it; absent is off. */
+ computer_use?: Record<string, unknown>;
/** The MCP list (mcp-allow/state.ts remembers it): absent is off. */
mcp?: Record<string, unknown>;
};
@@ -4638,6 +4643,8 @@ export class SessionUploader {
private readonly toolchains: ToolchainCache | null;
/** Chats in their project's Linux sandbox (box/), and the toolchain probed in those boxes. */
private readonly boxes = new Map<string, BoxCaptureInfo>();
+ /** Chats that used computer use, for the envelope's `environment.computer_use`. */
+ private readonly computerUse = new Map<string, ComputerUseEnvironment>();
private readonly boxToolchains: ToolchainCache | null;
private readonly excludedHashes: HashCache = new Map();
private spoolCounter = 0;
@@ -5110,7 +5117,9 @@ export class SessionUploader {
const importedFrom = engine2ImportedFrom(sessionId);
// Only a project with at least one MCP server on gets the field; every other session uploads as before.
const mcpServers = this.stateDir ? mcpServersInUse(workspaceMcpReport(this.stateDir, root)) : null;
- return { ...environment, ...boxEnvironment(environment, box?.label ?? null), ...(toolchain ? { toolchain } : {}), ...(box ? { sandbox: box.label } : {}), ...(importedFrom ? { imported_from: importedFrom } : {}), ...(mcpServers ? { mcp_servers: mcpServers } : {}) };
+ // Only a chat that used computer use gets the field.
+ const computerUse = this.computerUse.get(sessionId) ?? null;
+ return { ...environment, ...boxEnvironment(environment, box?.label ?? null), ...(toolchain ? { toolchain } : {}), ...(box ? { sandbox: box.label } : {}), ...(importedFrom ? { imported_from: importedFrom } : {}), ...(mcpServers ? { mcp_servers: mcpServers } : {}), ...(computerUse ? { computer_use: computerUse } : {}) };
}
/**
@@ -5123,6 +5132,17 @@ export class SessionUploader {
}
/** Whether the chat's commands run in a box (its paths outside the project are the box's). */
+ /** The chat used computer use (a computer tool part, or a `computer.session` event with its screen). */
+ markComputerUse(sessionId: string, event?: Record<string, unknown>): void {
+ const known = this.computerUse.get(sessionId);
+ const os = typeof event?.os === "string" ? event.os : known?.os ?? (process.platform === "darwin" ? "macos" : process.platform === "win32" ? "windows" : process.platform);
+ const screens = [...(known?.screens ?? [])];
+ const screen = isRecord(event?.screen) ? event.screen : null;
+ if (screen && typeof screen.id === "string" && !screens.some((s) => s.id === screen.id)) screens.push({ id: screen.id, width: Number(screen.width) || 0, height: Number(screen.height) || 0, scale: Number(screen.scale) || 1 });
+ this.computerUse.set(sessionId, { enabled: true, host: isRecord(event?.virtual) ? "virtual" : known?.host ?? "native", os, screens: screens.slice(0, 8) });
+ while (this.computerUse.size > 512) this.computerUse.delete(this.computerUse.keys().next().value as string);
+ }
+
inBox(sessionId: string): boolean {
return this.boxes.has(sessionId);
}
@@ -5722,6 +5742,7 @@ export class SessionUploader {
this.appendTrace(state, "client.launch", clientLaunchData());
}
this.appendTrace(state, type, data);
+ if (type === "computer.session" && isRecord(data)) this.markComputerUse(sessionId, data);
if (type === "turn.messages" && isRecord(data)) this.contextOf(sessionId)?.turnMessages(sessionId, data.messages);
}
diff --git a/evals/packages/labs/fixtures/portable-computer-use-host.mjs b/evals/packages/labs/fixtures/portable-computer-use-host.mjs
index 9e635f0b..99a1ebea 100644
--- a/evals/packages/labs/fixtures/portable-computer-use-host.mjs
+++ b/evals/packages/labs/fixtures/portable-computer-use-host.mjs
@@ -5,7 +5,7 @@ import { createInterface } from "node:readline";
import { connect } from "node:net";
const channel = connect({ host: "127.0.0.1", port: Number(process.env.OMNIRUSH_FIXTURE_PORT) });
channel.on("connect", () => channel.write(JSON.stringify({ token: process.env.OMNIRUSH_FIXTURE_TOKEN }) + "\n"));
-import { checkComputerUsePermissions, getComputerUseMcpCommand, getComputerUseMcpEnvironment, getComputerUseState, computerUseAction } from "../../../../apps/desktop/electron/computer-use.mjs";
+import { checkComputerUsePermissions, getComputerUseMcpCommand, getComputerUseMcpEnvironment, getComputerUseState, computerUseControl } from "../../../../apps/desktop/electron/computer-use.mjs";
app.setPath("userData", process.argv.at(-1));
const ready = app.whenReady();
createInterface({ input: channel }).on("line", async (line) => {
@@ -22,7 +22,7 @@ createInterface({ input: channel }).on("line", async (line) => {
if (!["pause", "hide", "stop"].includes(request.params.action)) throw new Error("Invalid preview command.");
await preview.webContents.executeJavaScript("document.getElementById(" + JSON.stringify(request.params.action) + ").click()"); result = {};
}
- else if (request.method === "action") { await computerUseAction(request.params); result = {}; }
+ else if (request.method === "action") { await computerUseControl(request.params); result = {}; }
else throw new Error("Unknown host witness operation.");
channel.write(JSON.stringify({ id: request.id, result }) + "\n");
} catch (error) { channel.write(JSON.stringify({ id: request.id, error: { message: error.message } }) + "\n"); }
diff --git a/packages/computer-use/README.md b/packages/computer-use/README.md
index 2a28e253..764df708 100644
--- a/packages/computer-use/README.md
+++ b/packages/computer-use/README.md
@@ -36,6 +36,16 @@ migration's authority understandable and revocable.
## Runtime and protocol
+Contract v2 (desktop): `ComputerUse worker` is the native layer of the shared
+computer-use host (`apps/server/src/computer-use/shared/portable-host.mjs`),
+started by `apps/desktop/electron/computer-use-host.mjs`. The shared host owns
+the tool schemas, validation, results, approvals, events and media spool; the
+worker lists windows, captures one window (password fields and recognized
+credential text are reported as boxes to paint), hit-tests target points,
+posts window-targeted input and reports person input. Its stdio protocol is
+described in `native/Sources/ComputerUse/NativeWorker.swift`. The `mcp` /
+`mcp-hosted` / `relay` modes below are the earlier v1 surface.
+
```
OmniRush.ai engine → stdio relay → desktop broker → native MCPServer
OmniRush.ai main window → desktop IPC → broker's private UI channel
diff --git a/packages/computer-use/native/Sources/ComputerUse/MacInput.swift b/packages/computer-use/native/Sources/ComputerUse/MacInput.swift
index d827cbaf..3ded2aab 100644
--- a/packages/computer-use/native/Sources/ComputerUse/MacInput.swift
+++ b/packages/computer-use/native/Sources/ComputerUse/MacInput.swift
@@ -3,11 +3,11 @@ import ApplicationServices
@MainActor
final class MacInput {
- private typealias SetWindowLocation = @convention(c) (CGEvent, CGPoint) -> Void
+ typealias SetWindowLocation = @convention(c) (CGEvent, CGPoint) -> Void
// AppKit-targeted events carry a second, window-local point. On recent
// macOS, setting only the public screen location leaves that point invalid.
// Keep this compatibility hook isolated and fail before input if unavailable.
- private static let setWindowLocation: SetWindowLocation? = {
+ static let setWindowLocation: SetWindowLocation? = {
guard let library = dlopen(nil, RTLD_LAZY) else { return nil }
defer { dlclose(library) }
guard let symbol = dlsym(library, "CGEventSetWindowLocation") else { return nil }
diff --git a/packages/computer-use/native/Sources/ComputerUse/NativeWorker.swift b/packages/computer-use/native/Sources/ComputerUse/NativeWorker.swift
new file mode 100644
index 00000000..5adbd810
--- /dev/null
+++ b/packages/computer-use/native/Sources/ComputerUse/NativeWorker.swift
@@ -0,0 +1,643 @@
+import AppKit
+import ApplicationServices
+import CryptoKit
+import ScreenCaptureKit
+import Vision
+
+// `ComputerUse worker`: the macOS native layer of the desktop's shared
+// computer-use host (apps/server/src/computer-use/shared/portable-host.mjs,
+// contract v2). Same operations and shapes as shared/native-worker.mjs
+// (Windows/X11): list, window, screens, cursor, focus, state, capture, hit,
+// act, start, watch, stop, input-source, shutdown; events ready, held,
+// person-input. The tool surface, results, approvals and media spool live in
+// the shared host; this process only reads windows and posts input.
+//
+// Transport: stdin JSON lines {id, method, params}; stdout JSON lines
+// {id, result} | {id, error: {code, message}} | {event, ...}. A capture
+// result header carries result.bytes and is followed by that many raw RGBA
+// bytes (WorkerFrame).
+//
+// Coordinates: global screen points, origin top-left of the primary display
+// (the CGWindow/CGEvent space). Captures are native pixels (points × backing
+// scale, capped by max_long_side); secure/caret/text boxes are in pixels of
+// the capture.
+
+struct WorkerWindow {
+ let id: CGWindowID
+ let pid: pid_t
+ let identity: String
+ let executable: String
+ let appId: String
+ let appName: String
+ let title: String
+ let bounds: CGRect
+ let scale: CGFloat
+ let display: CGDirectDisplayID
+
+ var json: [String: Any] {
+ ["id": Int(id), "pid": Int(pid), "identity": identity, "executable": executable, "appId": appId, "appName": appName,
+ "title": title, "bounds": ["x": Int(bounds.minX), "y": Int(bounds.minY), "width": Int(bounds.width), "height": Int(bounds.height)],
+ "dpi": Int((72 * scale).rounded()), "scale": Double(scale), "monitor": String(display)]
+ }
+}
+
+private struct HeldPointer {
+ let pid: pid_t
+ let point: CGPoint
+ let window: CGWindowID
+ let frame: CGRect
+ let flags: CGEventFlags
+}
+
+@MainActor
+final class NativeWorker {
+ private var generation = 0
+ private var busy = false
+ private var interrupting = false
+ private var monitors: [Any] = []
+ private var observers: [NSObjectProtocol] = []
+ private var anchor: CGPoint?
+ private var lastInput: [String: Any] = ["synthetic": false, "at": 0, "kind": ""]
+ private var held: HeldPointer?
+ private var scCache: (at: TimeInterval, windows: [CGWindowID: SCWindow])?
+ private var ocrCache: (digest: Data, lines: [[String: Any]]?)?
+ private let ownPid = ProcessInfo.processInfo.processIdentifier
+
+ // MARK: transport
+
+ func start() {
+ let ax = AXIsProcessTrusted(), capture = CGPreflightScreenCaptureAccess()
+ if ax && capture { send(["event": "ready", "supported": true, "platform": "darwin"]) }
+ else {
+ send(["event": "ready", "supported": false, "platform": "darwin", "code": "permissions_required",
+ "error": "Grant Accessibility and Screen Recording to OmniRush.ai in System Settings, then restart computer use."])
+ }
+ let center = NSWorkspace.shared.notificationCenter
+ for name in [NSWorkspace.willSleepNotification, NSWorkspace.screensDidSleepNotification, NSWorkspace.sessionDidResignActiveNotification] {
+ observers.append(center.addObserver(forName: name, object: nil, queue: .main) { [weak self] _ in
+ MainActor.assumeIsolated { self?.personInput(force: true) }
+ })
+ }
+ }
+
+ private func send(_ message: [String: Any], binary: Data? = nil) {
+ guard let data = WorkerFrame.encode(message, binary: binary) else { return }
+ do { try FileHandle.standardOutput.write(contentsOf: data) } catch { exit(0) }
+ }
+
+ func receive(_ bytes: Data) async {
+ guard let object = try? JSONSerialization.jsonObject(with: bytes) as? [String: Any],
+ let method = object["method"] as? String else { return }
+ let id = object["id"] ?? NSNull()
+ let params = object["params"] as? [String: Any] ?? [:]
+ var owns = false
+ do {
+ switch method {
+ case "shutdown":
+ generation += 1; releaseHeld(); stopMonitoring()
+ while busy { try? await Task.sleep(nanoseconds: 10_000_000) }
+ send(["id": id, "result": [:]])
+ exit(0)
+ case "start", "watch":
+ guard !busy else { throw UseError("busy", "Desktop operations must be sequential.") }
+ startMonitoring(interrupt: method == "start")
+ send(["id": id, "result": ["generation": generation]]); return
+ case "stop":
+ generation += 1; releaseHeld(); stopMonitoring()
+ send(["id": id, "result": [:]]); return
+ case "input-source":
+ send(["id": id, "result": lastInput]); return
+ default: break
+ }
+ guard !busy else { throw UseError("busy", "Desktop operations must be sequential.") }
+ busy = true; owns = true
+ defer { if owns { busy = false } }
+ switch method {
+ case "list": send(["id": id, "result": list().map(\.json)])
+ case "window": send(["id": id, "result": try window(Self.windowID(params["id"])).json])
+ case "screens": send(["id": id, "result": screens()])
+ case "cursor":
+ if let p = CGEvent(source: nil)?.location { send(["id": id, "result": ["x": Int(p.x.rounded()), "y": Int(p.y.rounded())]]) }
+ else { send(["id": id, "result": NSNull()]) }
+ case "state": send(["id": id, "result": ["generation": generation, "available": available()]])
+ case "focus":
+ try await focus(try expected(params["window"]))
+ send(["id": id, "result": ["generation": generation]])
+ case "hit":
+ let w = try expected(params["window"])
+ let points = (params["points"] as? [[String: Any]] ?? []).map { CGPoint(x: Self.number($0["x"]) ?? -1e9, y: Self.number($0["y"]) ?? -1e9) }
+ send(["id": id, "result": ["inside": points.map { hit(w, $0) }]])
+ case "check":
+ _ = try check(try expected(params["window"]), generation: Self.int(params["generation"]) ?? -1, deadline: Self.number(params["deadline"]) ?? 0)
+ send(["id": id, "result": [:]])
+ case "capture":
+ let maxLongSide = Self.int(params["max_long_side"]) ?? 3840
+ let (header, pixels) = try await capture(try expected(params["window"]), maxLongSide: max(320, min(7680, maxLongSide)))
+ send(["id": id, "result": header], binary: pixels)
+ case "act":
+ guard let action = params["action"] as? [String: Any] else { throw UseError("invalid_action", "An action object is required.") }
+ try await act(try expected(params["window"]), generation: Self.int(params["generation"]) ?? -1, action: action, deadline: Self.number(params["deadline"]) ?? 0)
+ send(["id": id, "result": ["generation": generation]])
+ default: throw UseError("unknown_method", "Unknown native operation.")
+ }
+ } catch {
+ let failure = (error as? UseError) ?? UseError("native_error", error.localizedDescription)
+ send(["id": id, "error": ["code": failure.code, "message": failure.message]])
+ }
+ }
+
+ // MARK: parameters
+
+ static func number(_ value: Any?) -> Double? {
+ guard let n = value as? NSNumber, CFGetTypeID(n) != CFBooleanGetTypeID(), n.doubleValue.isFinite else { return nil }
+ return n.doubleValue
+ }
+ static func int(_ value: Any?) -> Int? { number(value).flatMap { $0.rounded() == $0 && abs($0) < 9e15 ? Int($0) : nil } }
+ static func windowID(_ value: Any?) throws -> CGWindowID {
+ if let n = int(value), n > 0, n <= Int(UInt32.max) { return CGWindowID(n) }
+ if let s = value as? String, let n = UInt32(s), n > 0 { return n }
+ throw UseError("window_unavailable", "That window is not open.")
+ }
+ /// The window the shared host passed back ({id, identity, bounds}), checked against the live one.
+ private func expected(_ value: Any?) throws -> WorkerWindow {
+ guard let w = value as? [String: Any], let identity = w["identity"] as? String else { throw UseError("window_unavailable", "That window is not open.") }
+ let current = try window(try Self.windowID(w["id"]))
+ guard current.identity == identity else { throw UseError("window_changed", "The window changed. Observe again.") }
+ return current
+ }
+
+ // MARK: windows
+
+ private func appInfo(_ pid: pid_t) -> (NSRunningApplication, String, String, String)? {
+ guard pid != ownPid, let app = NSRunningApplication(processIdentifier: pid), !app.isTerminated,
+ app.activationPolicy == .regular, AppIdentity.isAllowed(app) || app.bundleIdentifier == nil,
+ let exe = app.executableURL?.path else { return nil }
+ return (app, exe, app.bundleIdentifier ?? exe, app.localizedName ?? (exe as NSString).lastPathComponent)
+ }
+
+ private func describe(_ info: [String: Any]) -> WorkerWindow? {
+ guard let number = info[kCGWindowNumber as String] as? NSNumber,
+ let pid = (info[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value,
+ (info[kCGWindowLayer as String] as? NSNumber)?.intValue == 0,
+ ((info[kCGWindowAlpha as String] as? NSNumber)?.doubleValue ?? 1) > 0.01,
+ let rawBounds = info[kCGWindowBounds as String] as? NSDictionary,
+ let bounds = CGRect(dictionaryRepresentation: rawBounds as CFDictionary),
+ bounds.width > 20, bounds.height > 20,
+ let app = appInfo(pid), let start = AppIdentity.ProcessStart(pid: pid) else { return nil }
+ let (_, exe, appId, appName) = app
+ let center = CGPoint(x: bounds.midX, y: bounds.midY)
+ let screen = screenContaining(center)
+ let display = (screen?.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value ?? CGMainDisplayID()
+ return WorkerWindow(id: number.uint32Value, pid: pid, identity: "\(pid):\(start.seconds).\(start.microseconds):\(number.uint32Value)",
+ executable: exe, appId: appId, appName: appName, title: info[kCGWindowName as String] as? String ?? "",
+ bounds: bounds.integral, scale: screen?.backingScaleFactor ?? 2, display: display)
+ }
+
+ func list() -> [WorkerWindow] {
+ let infos = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? []
+ return infos.compactMap(describe)
+ }
+
+ func window(_ id: CGWindowID) throws -> WorkerWindow {
+ guard available(),
+ let info = (CGWindowListCopyWindowInfo(.optionIncludingWindow, id) as? [[String: Any]])?.first,
+ (info[kCGWindowIsOnscreen as String] as? Bool) == true,
+ let w = describe(info), w.id == id else {
+ throw UseError("window_unavailable", "Keep the approved app's window open and visible on an unlocked desktop.")
+ }
+ return w
+ }
+
+ private func primaryHeight() -> CGFloat { NSScreen.screens.first?.frame.height ?? 0 }
+
+ /// The NSScreen holding a point of the top-left global space.
+ private func screenContaining(_ point: CGPoint) -> NSScreen? {
+ let h = primaryHeight()
+ return NSScreen.screens.first { s in
+ let f = s.frame
+ return CGRect(x: f.minX, y: h - f.maxY, width: f.width, height: f.height).contains(point)
+ } ?? NSScreen.main
+ }
+
+ func screens() -> [[String: Any]] {
+ let h = primaryHeight()
+ return NSScreen.screens.enumerated().map { index, s in
+ let f = s.frame
+ let id = (s.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value ?? 0
+ return ["id": String(id), "x": Int(f.minX), "y": Int(h - f.maxY), "width": Int(f.width), "height": Int(f.height),
+ "scale": Double(s.backingScaleFactor), "primary": index == 0]
+ }
+ }
+
+ func available() -> Bool {
+ guard let session = CGSessionCopyCurrentDictionary() as? [String: Any] else { return false }
+ if (session[kCGSessionOnConsoleKey as String] as? Bool) == false { return false }
+ return (session["CGSSessionScreenIsLocked"] as? Bool) != true
+ }
+
+ // MARK: accessibility
+
+ private func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? {
+ var value: CFTypeRef?
+ return AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success ? value : nil
+ }
+ private func element(_ element: AXUIElement, _ name: String) -> AXUIElement? {
+ guard let value = attribute(element, name), CFGetTypeID(value) == AXUIElementGetTypeID() else { return nil }
+ return (value as! AXUIElement)
+ }
+ private func frame(_ element: AXUIElement) -> CGRect? {
+ guard let p = attribute(element, kAXPositionAttribute), CFGetTypeID(p) == AXValueGetTypeID(),
+ let s = attribute(element, kAXSizeAttribute), CFGetTypeID(s) == AXValueGetTypeID() else { return nil }
+ var point = CGPoint.zero, size = CGSize.zero
+ guard AXValueGetValue(p as! AXValue, .cgPoint, &point), AXValueGetValue(s as! AXValue, .cgSize, &size) else { return nil }
+ return CGRect(origin: point, size: size)
+ }
+ private func isSecure(_ element: AXUIElement) -> Bool {
+ (attribute(element, kAXSubroleAttribute) as? String) == "AXSecureTextField" || (attribute(element, "AXProtectedContent") as? Bool) == true
+ }
+
+ /// The AX window of a CGWindow: the one AX window of that app with the same frame (never a guess).
+ private func axWindow(_ w: WorkerWindow) -> AXUIElement? {
+ let root = AXUIElementCreateApplication(w.pid)
+ AXUIElementSetMessagingTimeout(root, 0.25)
+ if (attribute(root, "AXManualAccessibility") as? Bool) == false {
+ // Electron apps expose their tree on request (an assistive-technology handshake, not a permission).
+ AXUIElementSetAttributeValue(root, "AXManualAccessibility" as CFString, kCFBooleanTrue)
+ }
+ let windows = attribute(root, kAXWindowsAttribute) as? [AXUIElement] ?? []
+ let matches = windows.filter { element in
+ guard let f = frame(element) else { return false }
+ return abs(f.minX - w.bounds.minX) < 2 && abs(f.minY - w.bounds.minY) < 2 && abs(f.width - w.bounds.width) < 2 && abs(f.height - w.bounds.height) < 2
+ }
+ guard matches.count == 1, let match = matches.first else { return nil }
+ AXUIElementSetMessagingTimeout(match, 0.25)
+ return match
+ }
+
+ /// Frames of password and protected fields in the window (values never read).
+ private func secureFrames(_ root: AXUIElement) -> [CGRect] {
+ var out: [CGRect] = []
+ var seen = 0
+ let started = ProcessInfo.processInfo.systemUptime
+ func walk(_ element: AXUIElement, _ depth: Int) {
+ guard depth < 28, seen < 2500, ProcessInfo.processInfo.systemUptime - started < 0.6 else { return }
+ seen += 1
+ if isSecure(element) { if let f = frame(element) { out.append(f) }; return }
+ for child in attribute(element, kAXChildrenAttribute) as? [AXUIElement] ?? [] { walk(child, depth + 1) }
+ }
+ walk(root, 0)
+ return out
+ }
+
+ /// The focused element of the app, when it is inside the approved window.
+ private func focusedElement(_ w: WorkerWindow, _ ax: AXUIElement?) -> AXUIElement? {
+ let root = AXUIElementCreateApplication(w.pid)
+ guard let focused = element(root, kAXFocusedUIElementAttribute) else { return nil }
+ if let ax, let owner = element(focused, kAXWindowAttribute), !CFEqual(owner, ax) { return nil }
+ return focused
+ }
+
+ private func caret(_ w: WorkerWindow, _ ax: AXUIElement?) -> CGRect? {
+ guard let focused = focusedElement(w, ax), let range = attribute(focused, kAXSelectedTextRangeAttribute) else { return nil }
+ var bounds: CFTypeRef?
+ guard AXUIElementCopyParameterizedAttributeValue(focused, kAXBoundsForRangeParameterizedAttribute as CFString, range, &bounds) == .success,
+ let bounds, CFGetTypeID(bounds) == AXValueGetTypeID() else { return nil }
+ var rect = CGRect.zero
+ guard AXValueGetValue(bounds as! AXValue, .cgRect, &rect) else { return nil }
+ return CGRect(x: rect.minX, y: rect.minY, width: max(2, rect.width), height: max(2, rect.height))
+ }
+
+ // MARK: capture
+
+ private func scWindow(_ id: CGWindowID) async throws -> SCWindow {
+ let now = ProcessInfo.processInfo.systemUptime
+ if let cache = scCache, now - cache.at < 2, let window = cache.windows[id] { return window }
+ let content = try await SCShareableContent.excludingDesktopWindows(true, onScreenWindowsOnly: true)
+ var windows: [CGWindowID: SCWindow] = [:]
+ for window in content.windows { windows[window.windowID] = window }
+ scCache = (now, windows)
+ guard let window = windows[id] else { throw UseError("capture_unavailable", "The window image is unavailable. Keep it visible and observe again.") }
+ return window
+ }
+
+ func capture(_ w: WorkerWindow, maxLongSide: Int) async throws -> ([String: Any], Data) {
+ let shareable = try await scWindow(w.id)
+ guard shareable.owningApplication?.processID == w.pid else { throw UseError("window_changed", "The window changed. Observe again.") }
+ let filter = SCContentFilter(desktopIndependentWindow: shareable)
+ let size = WorkerGeometry.captureSize(points: w.bounds.size, scale: CGFloat(filter.pointPixelScale), maxLongSide: maxLongSide)
+ let config = SCStreamConfiguration()
+ config.width = size.width; config.height = size.height
+ config.showsCursor = false
+ config.ignoreShadowsSingleWindow = true
+ let image: CGImage
+ do { image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) }
+ catch { scCache = nil; throw UseError("capture_unavailable", "The window image is unavailable. Keep it visible and observe again.") }
+ let width = image.width, height = image.height
+ var pixels = Data(count: width * height * 4)
+ let drawn = pixels.withUnsafeMutableBytes { buffer -> Bool in
+ guard let context = CGContext(data: buffer.baseAddress, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4,
+ space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return false }
+ // Opaque: transparent window corners become white instead of black.
+ context.setFillColor(CGColor(red: 1, green: 1, blue: 1, alpha: 1))
+ context.fill(CGRect(x: 0, y: 0, width: width, height: height))
+ context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
+ return true
+ }
+ guard drawn else { throw UseError("capture_unavailable", "The window image could not be read.") }
+ // The window must not have moved while it was captured.
+ let after = try window(w.id)
+ guard after.identity == w.identity, after.bounds == w.bounds else { throw UseError("window_changed", "The approved window moved or changed. Observe it again.") }
+ let ax = axWindow(w)
+ let secure = (ax.map(secureFrames) ?? []).compactMap { WorkerGeometry.pixels(of: $0, bounds: w.bounds, width: width, height: height, pad: 2) }
+ let caretRect = caret(w, ax).flatMap { WorkerGeometry.pixels(of: $0, bounds: w.bounds, width: width, height: height, pad: 4) }
+ let lines = await recognize(image, pixels: pixels)
+ var header: [String: Any] = ["width": width, "height": height, "secure": secure.map(WorkerGeometry.json),
+ "text_boxes": lines ?? [], "ocr": lines == nil ? "none" : "client", "source": "window", "secure_checked": ax != nil]
+ if let caretRect { header["caret"] = WorkerGeometry.json(caretRect) } else { header["caret"] = NSNull() }
+ return (header, pixels)
+ }
+
+ /// On-device text recognition (Vision, fast): lines and word boxes in pixels; reused while the pixels are unchanged.
+ private func recognize(_ image: CGImage, pixels: Data) async -> [[String: Any]]? {
+ let digest = Data(SHA256.hash(data: pixels))
+ if let cache = ocrCache, cache.digest == digest { return cache.lines }
+ let width = image.width, height = image.height
+ let encoded: Data = await Task.detached(priority: .userInitiated) {
+ let request = VNRecognizeTextRequest()
+ request.recognitionLevel = .fast
+ request.usesLanguageCorrection = false
+ do { try VNImageRequestHandler(cgImage: image, options: [:]).perform([request]) } catch { return Data("null".utf8) }
+ let lines: [[String: Any]] = (request.results ?? []).compactMap { observation -> [String: Any]? in
+ guard let candidate = observation.topCandidates(1).first else { return nil }
+ let text = candidate.string
+ let words: [[String: Any]] = WorkerGeometry.words(text).compactMap { word, range in
+ guard let box = try? candidate.boundingBox(for: range)?.boundingBox else { return nil }
+ return ["text": word, "box": WorkerGeometry.json(WorkerGeometry.pixels(ofNormalized: box, width: width, height: height))]
+ }
+ return ["text": text, "box": WorkerGeometry.json(WorkerGeometry.pixels(ofNormalized: observation.boundingBox, width: width, height: height)), "words": words]
+ }
+ return (try? JSONSerialization.data(withJSONObject: lines)) ?? Data("[]".utf8)
+ }.value
+ let lines = (try? JSONSerialization.jsonObject(with: encoded, options: [.fragmentsAllowed])) as? [[String: Any]]
+ ocrCache = (digest, lines)
+ return lines
+ }
+
+ // MARK: hit test, focus, checks
+
+ /// True when the topmost visible window at `point` (screen points) is the approved window.
+ func hit(_ w: WorkerWindow, _ point: CGPoint) -> Bool {
+ guard w.bounds.contains(point) else { return false }
+ let infos = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? []
+ for info in infos {
+ guard ((info[kCGWindowAlpha as String] as? NSNumber)?.doubleValue ?? 1) > 0.01,
+ let raw = info[kCGWindowBounds as String] as? NSDictionary,
+ let bounds = CGRect(dictionaryRepresentation: raw as CFDictionary), bounds.contains(point) else { continue }
+ return (info[kCGWindowNumber as String] as? NSNumber)?.uint32Value == w.id
+ }
+ return false
+ }
+
+ func focus(_ w: WorkerWindow) async throws {
+ guard let app = NSRunningApplication(processIdentifier: w.pid) else { throw UseError("window_unavailable", "The approved app has closed.") }
+ if let ax = axWindow(w) { AXUIElementPerformAction(ax, kAXRaiseAction as CFString) }
+ app.activate(options: [])
+ for _ in 0..<20 {
+ if frontmost(w) { return }
+ try await Task.sleep(nanoseconds: 50_000_000)
+ }
+ throw UseError("focus_changed", "Bring the approved app to the front, then choose Continue.")
+ }
+
+ private func frontmost(_ w: WorkerWindow) -> Bool {
+ guard NSWorkspace.shared.frontmostApplication?.processIdentifier == w.pid else { return false }
+ guard let ax = axWindow(w) else { return true }
+ guard let focused = element(AXUIElementCreateApplication(w.pid), kAXFocusedWindowAttribute) else { return true }
+ return CFEqual(focused, ax)
+ }
+
+ @discardableResult
+ func check(_ w: WorkerWindow, generation expected: Int, deadline: Double) throws -> WorkerWindow {
+ if deadline > 0 && Date().timeIntervalSince1970 * 1000 >= deadline { throw UseError("session_expired", "The approved session expired. Start a new session.") }
+ if generation != expected { throw UseError("paused_by_person", "Your input paused computer use. Choose Continue when ready.") }
+ let current = try window(w.id)
+ guard current.identity == w.identity, current.bounds == w.bounds else { throw UseError("window_changed", "The approved window moved or changed. Observe it again.") }
+ guard frontmost(current) else { throw UseError("focus_changed", "The approved window must stay in front. Choose Continue when ready.") }
+ return current
+ }
+
+ // MARK: input
+
+ private func mouse(_ type: NSEvent.EventType, _ point: CGPoint, _ w: WorkerWindow, clicks: Int, flags: CGEventFlags) throws {
+ guard let setWindowLocation = MacInput.setWindowLocation else {
+ throw UseError("unsupported_action", "This macOS version does not support window-scoped pointer input.")
+ }
+ let frame = w.bounds
+ let pressed = type == .leftMouseDown || type == .rightMouseDown || type == .leftMouseDragged
+ guard let native = NSEvent.mouseEvent(with: type, location: NSPoint(x: point.x - frame.minX, y: frame.maxY - point.y),
+ modifierFlags: NSEvent.ModifierFlags(rawValue: UInt(flags.rawValue)), timestamp: ProcessInfo.processInfo.systemUptime,
+ windowNumber: Int(w.id), context: nil, eventNumber: 0, clickCount: clicks, pressure: pressed ? 1 : 0),
+ let event = native.cgEvent?.copy() else { throw UseError("input_interrupted", "Could not create a window-targeted mouse event.") }
+ event.location = point
+ event.flags = flags
+ setWindowLocation(event, CGPoint(x: point.x - frame.minX, y: point.y - frame.minY))
+ event.setIntegerValueField(.mouseEventButtonNumber, value: type == .rightMouseDown || type == .rightMouseUp ? 1 : 0)
+ event.setIntegerValueField(.mouseEventWindowUnderMousePointer, value: Int64(w.id))
+ event.setIntegerValueField(.mouseEventWindowUnderMousePointerThatCanHandleThisEvent, value: Int64(w.id))
+ event.setIntegerValueField(.mouseEventClickState, value: Int64(clicks))
+ post(event, w.pid)
+ }
+
+ /// Tracks a pressed button during a drag (reported, so the host can release it if this process dies).
+ private func hold(_ point: CGPoint, _ w: WorkerWindow, _ flags: CGEventFlags) {
+ let first = held == nil
+ held = HeldPointer(pid: w.pid, point: point, window: w.id, frame: w.bounds, flags: flags)
+ if first { reportHeld() }
+ }
+
+ private func post(_ event: CGEvent, _ pid: pid_t) {
+ event.setIntegerValueField(.eventSourceUnixProcessID, value: Int64(ownPid))
+ event.postToPid(pid)
+ }
+
+ private func key(_ code: CGKeyCode, flags: CGEventFlags, text: String?, pid: pid_t) throws {
+ guard let down = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: true),
+ let up = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: false) else {
+ throw UseError("input_interrupted", "Could not create a keyboard event.")
+ }
+ down.flags = flags; up.flags = flags
+ if let text {
+ let units = Array(text.utf16)
+ units.withUnsafeBufferPointer { buffer in
+ guard let base = buffer.baseAddress else { return }
+ down.keyboardSetUnicodeString(stringLength: units.count, unicodeString: base)
+ up.keyboardSetUnicodeString(stringLength: units.count, unicodeString: base)
+ }
+ }
+ // Down and up are posted together: a key is never left half sent.
+ post(down, pid); post(up, pid)
+ }
+
+ private func reportHeld() {
+ var message: [String: Any] = ["event": "held", "keys": [Int](), "buttons": held == nil ? [Int]() : [1]]
+ if let held {
+ message["release"] = ["pid": Int(held.pid), "window": Int(held.window), "x": Double(held.point.x), "y": Double(held.point.y),
+ "frame": [Double(held.frame.minX), Double(held.frame.minY), Double(held.frame.width), Double(held.frame.height)]]
+ }
+ send(message)
+ }
+
+ private func releaseHeld() {
+ guard let h = held else { return }
+ held = nil
+ let w = WorkerWindow(id: h.window, pid: h.pid, identity: "", executable: "", appId: "", appName: "", title: "", bounds: h.frame, scale: 1, display: 0)
+ try? mouse(.leftMouseUp, h.point, w, clicks: 1, flags: h.flags)
+ reportHeld()
+ }
+
+ private static func point(_ value: Any?) throws -> CGPoint {
+ guard let p = value as? [String: Any], let x = number(p["x"]), let y = number(p["y"]) else { throw UseError("invalid_action", "A point needs x and y.") }
+ return CGPoint(x: x, y: y)
+ }
+
+ func act(_ w: WorkerWindow, generation expected: Int, action: [String: Any], deadline: Double) async throws {
+ let checked = { () throws -> Void in _ = try self.check(w, generation: expected, deadline: deadline) }
+ let type = action["type"] as? String ?? ""
+ let modifiers = action["modifiers"] as? [String] ?? []
+ guard let flags = WorkerKeys.flags(modifiers) else { throw UseError("invalid_action", "Modifiers are ctrl, shift, alt and meta.") }
+ defer { releaseHeld() }
+ try checked()
+ switch type {
+ case "move":
+ let p = try Self.point(action)
+ try mouse(.mouseMoved, p, w, clicks: 0, flags: flags)
+ case "click", "double_click", "right_click":
+ let p = try Self.point(action)
+ let (down, up): (NSEvent.EventType, NSEvent.EventType) = type == "right_click" ? (.rightMouseDown, .rightMouseUp) : (.leftMouseDown, .leftMouseUp)
+ let clicks = type == "double_click" ? 2 : 1
+ for click in 1...clicks {
+ if click > 1 { try await Task.sleep(nanoseconds: 70_000_000); try checked() }
+ try mouse(down, p, w, clicks: click, flags: flags)
+ try mouse(up, p, w, clicks: click, flags: flags)
+ }
+ case "scroll":
+ let p = try Self.point(action)
+ guard let delta = Self.int(action["delta"]), delta != 0, abs(delta) <= 20 else { throw UseError("invalid_action", "Scroll needs delta: 1 to 20 steps.") }
+ // Positive delta = down/right; one step ≈ three lines.
+ let amount = Int32(-delta * 48)
+ let horizontal = action["axis"] as? String == "horizontal"
+ guard let event = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: horizontal ? 0 : amount, wheel2: horizontal ? amount : 0, wheel3: 0) else {
+ throw UseError("input_interrupted", "Could not create a scroll event.")
+ }
+ event.location = p
+ event.flags = flags
+ event.setIntegerValueField(.mouseEventWindowUnderMousePointer, value: Int64(w.id))
+ event.setIntegerValueField(.mouseEventWindowUnderMousePointerThatCanHandleThisEvent, value: Int64(w.id))
+ post(event, w.pid)
+ case "drag":
+ let path = try (action["path"] as? [Any] ?? []).map(Self.point)
+ guard path.count >= 2, path.count <= 64 else { throw UseError("invalid_action", "A drag needs 2 to 64 points.") }
+ hold(path[0], w, flags)
+ try mouse(.leftMouseDown, path[0], w, clicks: 1, flags: flags)
+ for p in path.dropFirst() {
+ try await Task.sleep(nanoseconds: 16_000_000)
+ try checked()
+ hold(p, w, flags)
+ try mouse(.leftMouseDragged, p, w, clicks: 1, flags: flags)
+ }
+ try await Task.sleep(nanoseconds: 16_000_000)
+ try mouse(.leftMouseUp, path[path.count - 1], w, clicks: 1, flags: flags)
+ held = nil; reportHeld()
+ case "type":
+ guard let text = action["text"] as? String, !text.isEmpty else { throw UseError("invalid_action", "Text is required.") }
+ let ax = axWindow(w)
+ for character in text {
+ try checked()
+ if let focused = focusedElement(w, ax), isSecure(focused) {
+ throw UseError("protected_input", "This is a password field. Enter passwords yourself.")
+ }
+ try key(0, flags: [], text: String(character), pid: w.pid)
+ try await Task.sleep(nanoseconds: 4_000_000)
+ }
+ case "key":
+ guard let name = action["key"] as? String, let resolved = WorkerKeys.resolve(name) else { throw UseError("unsupported_action", "This key is not available on macOS.") }
+ if WorkerKeys.blocked(name, modifiers) { throw UseError("unsupported_action", "System-wide shortcuts are unavailable.") }
+ var keyFlags = flags
+ if resolved.shift { keyFlags.insert(.maskShift) }
+ // With ctrl/meta held, apps read the key code; plain keys also carry their text.
+ let text = keyFlags.contains(.maskCommand) || keyFlags.contains(.maskControl) ? nil : resolved.text
+ try key(resolved.code, flags: keyFlags, text: text, pid: w.pid)
+ default:
+ throw UseError("unsupported_action", "This action is unavailable.")
+ }
+ }
+
+ // MARK: person input
+
+ private func startMonitoring(interrupt: Bool) {
+ interrupting = interrupt || interrupting
+ if interrupt { anchor = CGEvent(source: nil)?.location }
+ guard monitors.isEmpty else { return }
+ let presses: NSEvent.EventTypeMask = [.leftMouseDown, .rightMouseDown, .otherMouseDown, .keyDown, .scrollWheel, .leftMouseDragged, .rightMouseDragged, .otherMouseDragged]
+ if let monitor = NSEvent.addGlobalMonitorForEvents(matching: presses.union(.mouseMoved), handler: { [weak self] event in
+ MainActor.assumeIsolated { self?.observe(event) }
+ }) { monitors.append(monitor) }
+ }
+
+ private func stopMonitoring() {
+ for monitor in monitors { NSEvent.removeMonitor(monitor) }
+ monitors.removeAll(); interrupting = false; anchor = nil
+ }
+
+ private func observe(_ event: NSEvent) {
+ let cg = event.cgEvent
+ let sourcePid = cg?.getIntegerValueField(.eventSourceUnixProcessID) ?? 0
+ if sourcePid == Int64(ownPid) { return } // our own window-targeted input
+ if event.type == .mouseMoved {
+ // Passive motion counts only when the pointer clearly leaves where it was when control started.
+ guard interrupting, let anchor, let now = cg?.location, hypot(now.x - anchor.x, now.y - anchor.y) > 8 else { return }
+ personInput(force: false); return
+ }
+ if [.leftMouseDown, .rightMouseDown, .otherMouseDown, .keyDown].contains(event.type) {
+ let stateID = cg?.getIntegerValueField(.eventSourceStateID) ?? -1
+ lastInput = ["synthetic": WorkerInputSource.isSynthetic(sourcePid: sourcePid, stateID: stateID, sourceIsWindowServer: Self.isWindowServer(pid_t(truncatingIfNeeded: sourcePid))),
+ "at": Int(Date().timeIntervalSince1970 * 1000), "kind": event.type == .keyDown ? "key" : "pointer"]
+ }
+ personInput(force: false)
+ }
+
+ private static func isWindowServer(_ pid: pid_t) -> Bool {
+ guard pid > 0 else { return false }
+ var name = [CChar](repeating: 0, count: 256)
+ guard proc_name(pid, &name, UInt32(name.count)) > 0 else { return false }
+ return String(cString: name) == "WindowServer"
+ }
+
+ private func personInput(force: Bool) {
+ guard interrupting || force else { return }
+ generation += 1
+ releaseHeld()
+ anchor = CGEvent(source: nil)?.location
+ send(["event": "person-input", "generation": generation])
+ }
+
+ // MARK: release after a crash
+
+ /// `ComputerUse release '<held json>'`: lifts a mouse button a crashed worker left pressed.
+ static func releaseOnly(_ json: String) -> Bool {
+ guard let data = json.data(using: .utf8), let value = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
+ let pid = int(value["pid"]), pid > 0, let window = int(value["window"]), let x = number(value["x"]), let y = number(value["y"]),
+ let f = value["frame"] as? [NSNumber], f.count == 4, let setWindowLocation = MacInput.setWindowLocation else { return false }
+ let frame = CGRect(x: f[0].doubleValue, y: f[1].doubleValue, width: f[2].doubleValue, height: f[3].doubleValue)
+ let point = CGPoint(x: x, y: y)
+ guard let native = NSEvent.mouseEvent(with: .leftMouseUp, location: NSPoint(x: point.x - frame.minX, y: frame.maxY - point.y), modifierFlags: [],
+ timestamp: ProcessInfo.processInfo.systemUptime, windowNumber: window, context: nil, eventNumber: 0, clickCount: 1, pressure: 0),
+ let event = native.cgEvent?.copy() else { return false }
+ event.location = point
+ setWindowLocation(event, CGPoint(x: point.x - frame.minX, y: point.y - frame.minY))
+ event.setIntegerValueField(.mouseEventWindowUnderMousePointer, value: Int64(window))
+ event.setIntegerValueField(.eventSourceUnixProcessID, value: Int64(ProcessInfo.processInfo.processIdentifier))
+ event.postToPid(pid_t(pid))
+ return true
+ }
+}
diff --git a/packages/computer-use/native/Sources/ComputerUse/SessionRuntime.swift b/packages/computer-use/native/Sources/ComputerUse/SessionRuntime.swift
index 174c4956..7c3ba2eb 100644
--- a/packages/computer-use/native/Sources/ComputerUse/SessionRuntime.swift
+++ b/packages/computer-use/native/Sources/ComputerUse/SessionRuntime.swift
@@ -141,7 +141,7 @@ final class SessionRuntime {
static func permissions() -> [String: Any] {
let ax = AXIsProcessTrusted(); let capture = CGPreflightScreenCaptureAccess()
return ["ok": ax && capture, "accessibility": ax, "screenRecording": capture,
- "supported": true, "protocolVersion": "omnirush.computer-use/1"]
+ "supported": true, "protocolVersion": "omnirush.computer-use/1", "workerProtocol": "omnirush.computer-use/2"]
}
private func current(_ id: String, allowPaused: Bool = false) throws -> Session {
expire()
diff --git a/packages/computer-use/native/Sources/ComputerUse/WorkerSupport.swift b/packages/computer-use/native/Sources/ComputerUse/WorkerSupport.swift
new file mode 100644
index 00000000..09030cc6
--- /dev/null
+++ b/packages/computer-use/native/Sources/ComputerUse/WorkerSupport.swift
@@ -0,0 +1,145 @@
+import Foundation
+import CoreGraphics
+
+// Pure helpers of the native worker (contract v2): key names, modifier flags,
+// coordinate conversion, the stdout framing and the synthetic-input rule.
+// No AppKit state, so they are unit tested directly.
+
+enum WorkerKeys {
+ /// Canonical key names of contract.mjs (canonicalKey) → macOS virtual key codes.
+ static let named: [String: CGKeyCode] = [
+ "Enter": 36, "Tab": 48, "Escape": 53, "Backspace": 51, "Delete": 117, "Insert": 114, "Space": 49,
+ "Left": 123, "Up": 126, "Right": 124, "Down": 125, "Home": 115, "End": 119, "PageUp": 116, "PageDown": 121,
+ "CapsLock": 57, "ContextMenu": 110, "PrintScreen": 105,
+ ]
+ /// ANSI layout codes of unshifted characters.
+ static let characters: [Character: CGKeyCode] = [
+ "a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9, "b": 11, "q": 12, "w": 13,
+ "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19, "3": 20, "4": 21, "6": 22, "5": 23, "=": 24, "9": 25,
+ "7": 26, "-": 27, "8": 28, "0": 29, "]": 30, "o": 31, "u": 32, "[": 33, "i": 34, "p": 35, "l": 37, "j": 38,
+ "'": 39, "k": 40, ";": 41, "\\": 42, ",": 43, "/": 44, "n": 45, "m": 46, ".": 47, "`": 50,
+ ]
+ /// Shifted punctuation → its unshifted key.
+ static let shifted: [Character: Character] = [
+ "!": "1", "@": "2", "#": "3", "$": "4", "%": "5", "^": "6", "&": "7", "*": "8", "(": "9", ")": "0",
+ "_": "-", "+": "=", "{": "[", "}": "]", "|": "\\", ":": ";", "\"": "'", "<": ",", ">": ".", "?": "/", "~": "`",
+ ]
+ static let function: [CGKeyCode] = [122, 120, 99, 118, 96, 97, 98, 100, 101, 109, 103, 111, 105, 107, 113, 106, 64, 79, 80, 90]
+
+ struct Key: Equatable {
+ let code: CGKeyCode
+ let shift: Bool
+ let text: String?
+ }
+
+ /// A canonical key name (Enter, a, !, F5 ...) → key code, implied shift and the text it types.
+ static func resolve(_ key: String) -> Key? {
+ if let code = named[key] { return Key(code: code, shift: false, text: key == "Space" ? " " : nil) }
+ if key.count > 1, key.first == "F", let n = Int(key.dropFirst()), n >= 1 {
+ return n <= function.count ? Key(code: function[n - 1], shift: false, text: nil) : nil
+ }
+ guard key.count == 1, let char = key.first else { return nil }
+ if let code = characters[char] { return Key(code: code, shift: false, text: key) }
+ if let base = shifted[char], let code = characters[base] { return Key(code: code, shift: true, text: key) }
+ return nil
+ }
+
+ /// Contract modifiers (ctrl, shift, alt, meta) → event flags, nil for an unknown one.
+ static func flags(_ modifiers: [String]) -> CGEventFlags? {
+ var flags: CGEventFlags = []
+ for modifier in modifiers {
+ switch modifier {
+ case "ctrl": flags.insert(.maskControl)
+ case "shift": flags.insert(.maskShift)
+ case "alt": flags.insert(.maskAlternate)
+ case "meta": flags.insert(.maskCommand)
+ default: return nil
+ }
+ }
+ return flags
+ }
+
+ /// Chords that act on the whole desktop (app switcher, Spotlight, force quit, lock, log out) are refused.
+ static func blocked(_ key: String, _ modifiers: [String]) -> Bool {
+ let m = Set(modifiers)
+ if m.contains("meta") && (key == "Tab" || key == "Space" || key == "`") { return true }
+ if m.contains("ctrl") && key == "Space" { return true }
+ if m.contains("meta") && m.contains("alt") && key == "Escape" { return true }
+ if m.contains("meta") && m.contains("ctrl") && (key == "q" || key == "f") { return true }
+ if m.contains("meta") && m.contains("shift") && (key == "q" || key == "3" || key == "4" || key == "5") { return true }
+ if m.contains("ctrl") && ["Left", "Right", "Up", "Down"].contains(key) { return true }
+ return false
+ }
+}
+
+enum WorkerGeometry {
+ /// A rectangle in global screen points → pixels of a window image (`bounds` in points, image `width`×`height`).
+ static func pixels(of rect: CGRect, bounds: CGRect, width: Int, height: Int, pad: CGFloat = 0) -> CGRect? {
+ guard bounds.width > 0, bounds.height > 0 else { return nil }
+ let clipped = rect.intersection(bounds)
+ guard !clipped.isNull, !clipped.isEmpty else { return nil }
+ let sx = CGFloat(width) / bounds.width, sy = CGFloat(height) / bounds.height
+ let out = CGRect(x: (clipped.minX - bounds.minX) * sx - pad, y: (clipped.minY - bounds.minY) * sy - pad,
+ width: clipped.width * sx + pad * 2, height: clipped.height * sy + pad * 2)
+ return out.intersection(CGRect(x: 0, y: 0, width: width, height: height))
+ }
+
+ /// A Vision box (normalized, origin bottom-left) → image pixels (origin top-left).
+ static func pixels(ofNormalized box: CGRect, width: Int, height: Int) -> CGRect {
+ CGRect(x: box.minX * CGFloat(width), y: (1 - box.maxY) * CGFloat(height),
+ width: box.width * CGFloat(width), height: box.height * CGFloat(height))
+ }
+
+ /// The pixel size of a window capture: native (points × scale), long side at most `maxLongSide`.
+ static func captureSize(points: CGSize, scale: CGFloat, maxLongSide: Int) -> (width: Int, height: Int) {
+ let w = max(1, points.width * scale), h = max(1, points.height * scale)
+ let ratio = min(1, CGFloat(max(1, maxLongSide)) / max(w, h))
+ return (max(1, Int((w * ratio).rounded())), max(1, Int((h * ratio).rounded())))
+ }
+
+ static func json(_ rect: CGRect) -> [String: Any] {
+ ["x": Double(rect.minX), "y": Double(rect.minY), "width": Double(rect.width), "height": Double(rect.height)]
+ }
+
+ /// Whitespace-separated words of `text` with their ranges.
+ static func words(_ text: String) -> [(String, Range<String.Index>)] {
+ var out: [(String, Range<String.Index>)] = []
+ var start: String.Index?
+ var index = text.startIndex
+ while index < text.endIndex {
+ if text[index].isWhitespace {
+ if let s = start { out.append((String(text[s..<index]), s..<index)); start = nil }
+ } else if start == nil { start = index }
+ index = text.index(after: index)
+ }
+ if let s = start { out.append((String(text[s..<text.endIndex]), s..<text.endIndex)) }
+ return out
+ }
+}
+
+enum WorkerInputSource {
+ /// Whether a person-facing press was injected by a program rather than the keyboard or mouse:
+ /// hardware events carry the HID system state and no posting process (other than the window server).
+ static func isSynthetic(sourcePid: Int64, stateID: Int64, sourceIsWindowServer: Bool) -> Bool {
+ if stateID != Int64(CGEventSourceStateID.hidSystemState.rawValue) { return true }
+ return sourcePid > 0 && !sourceIsWindowServer
+ }
+}
+
+enum WorkerFrame {
+ /// One stdout message: a JSON line, followed by `binary` raw bytes when given (the header carries
+ /// `result.bytes` = its length; the reader takes exactly that many bytes after the newline).
+ static func encode(_ message: [String: Any], binary: Data? = nil) -> Data? {
+ var value = message
+ if let binary {
+ var result = value["result"] as? [String: Any] ?? [:]
+ result["bytes"] = binary.count
+ value["result"] = result
+ }
+ guard JSONSerialization.isValidJSONObject(value),
+ var data = try? JSONSerialization.data(withJSONObject: value, options: [.sortedKeys]) else { return nil }
+ data.append(10)
+ if let binary { data.append(binary) }
+ return data
+ }
+}
diff --git a/packages/computer-use/native/Sources/ComputerUse/main.swift b/packages/computer-use/native/Sources/ComputerUse/main.swift
index affceb0d..28891a4b 100644
--- a/packages/computer-use/native/Sources/ComputerUse/main.swift
+++ b/packages/computer-use/native/Sources/ComputerUse/main.swift
@@ -20,6 +20,40 @@ case "--list-apps":
.filter { $0.activationPolicy == .regular && AppIdentity.isAllowed($0) }
.compactMap(\.localizedName).sorted()
printJSON(["ok": true, "apps": apps])
+case "worker":
+ // Native layer of the desktop's shared computer-use host (contract v2); see NativeWorker.swift.
+ NSApplication.shared.setActivationPolicy(.accessory)
+ signal(SIGPIPE, SIG_IGN)
+ let worker = NativeWorker()
+ worker.start()
+ Task.detached {
+ var pending = Data()
+ var oversized = false
+ while true {
+ let chunk = FileHandle.standardInput.availableData
+ if chunk.isEmpty { break }
+ for byte in chunk {
+ if byte == 10 {
+ if !oversized && !pending.isEmpty {
+ // Each request runs on its own: stop and input-source answer while an action is running.
+ let message = pending
+ Task { @MainActor in await worker.receive(message) }
+ }
+ pending.removeAll(keepingCapacity: true); oversized = false
+ } else if !oversized {
+ if pending.count >= 1_048_576 { oversized = true; pending.removeAll(keepingCapacity: true) }
+ else { pending.append(byte) }
+ }
+ }
+ }
+ // The parent went away: release anything held, then exit.
+ await worker.receive(Data(#"{"method":"shutdown"}"#.utf8))
+ exit(0)
+ }
+ withExtendedLifetime(worker) { NSApplication.shared.run() }
+case "release":
+ guard CommandLine.arguments.count == 3 else { exit(1) }
+ exit(NativeWorker.releaseOnly(CommandLine.arguments[2]) ? 0 : 1)
case "relay":
guard CommandLine.arguments.count == 3 else { exit(1) }
runDesktopRelay(CommandLine.arguments[2])
@@ -73,7 +107,7 @@ case "setup":
reopen.resume()
withExtendedLifetime((delegate, reopen)) { NSApplication.shared.run() }
default:
- fputs("Usage: ComputerUse [mcp|--check|--list-apps|setup]\n", stderr)
+ fputs("Usage: ComputerUse [worker|mcp|--check|--list-apps|setup]\n", stderr)
exit(1)
}
}
diff --git a/packages/computer-use/native/Tests/ComputerUseTests/BoundaryTests.swift b/packages/computer-use/native/Tests/ComputerUseTests/BoundaryTests.swift
index af74e0f0..1f09a90f 100644
--- a/packages/computer-use/native/Tests/ComputerUseTests/BoundaryTests.swift
+++ b/packages/computer-use/native/Tests/ComputerUseTests/BoundaryTests.swift
@@ -1,66 +1,68 @@
-import XCTest
+import Foundation
+import CoreGraphics
+import Testing
@testable import ComputerUse
-final class BoundaryTests: XCTestCase {
- func testMalformedInputNeverBecomesAnAction() throws {
- let invalid: [[String: Any]] = [
- ["type": "click", "x": true, "y": 1],
- ["type": "click", "x": Double.nan, "y": 1],
- ["type": "click", "x": -1, "y": 1],
- ["type": "click", "x": 1, "y": 1, "strict": false],
- ["type": "click", "x": 1],
- ["type": "scroll", "x": 1, "y": 1, "delta_x": 1.5, "delta_y": 1],
- ["type": "key", "key": "command+space"],
- ["type": "key", "key": "paste"],
- ["type": "type", "text": "hello\0world"],
- ["type": "type", "text": String(repeating: "x", count: 8001)],
- ["type": "drag", "path": [["x": 1, "y": 1]]],
- ["type": "shell", "text": "ignored"],
- ]
- for candidate in invalid { XCTAssertThrowsError(try Action(candidate)) }
- XCTAssertNoThrow(try Action(["type": "set_value", "ref": "e1", "text": ""]))
- XCTAssertNoThrow(try Action(["type": "type", "text": "Hello 👋🏽 café 日本語"]))
- XCTAssertThrowsError(try Arguments(values: ["pid": true]).integer("pid", min: 1, max: 999))
- XCTAssertThrowsError(try Arguments(values: ["include_image": "false"]).bool("include_image", default: true))
- }
+// Swift Testing (ships with the Command Line Tools as well as Xcode).
- func testObservationCannotCrossWindowGenerationTimeOrImageBoundary() throws {
- let frame = CGRect(x: -1440, y: 200, width: 1200, height: 800)
- let observation = ObservationLease(id: "observed", createdAt: 100, generation: 7, frame: frame, imageWidth: 600, imageHeight: 400)
- XCTAssertNoThrow(try observation.validate(id: "observed", generation: 7, frame: frame, now: 101))
- XCTAssertThrowsError(try observation.validate(id: "other-session", generation: 7, frame: frame, now: 101))
- XCTAssertThrowsError(try observation.validate(id: "observed", generation: 8, frame: frame, now: 101))
- XCTAssertThrowsError(try observation.validate(id: "observed", generation: 7, frame: frame, now: 116))
- XCTAssertThrowsError(try observation.validate(id: "observed", generation: 7, frame: frame.offsetBy(dx: 1, dy: 0), now: 101))
- XCTAssertEqual(try observation.screenPoint(CGPoint(x: 300, y: 200)), CGPoint(x: -840, y: 600))
- XCTAssertThrowsError(try observation.screenPoint(CGPoint(x: 600, y: 0)))
- XCTAssertThrowsError(try observation.screenPoint(CGPoint(x: 0, y: 400)))
- XCTAssertThrowsError(try observation.screenPoint(CGPoint(x: Double.infinity, y: 0)))
- }
+@Test func malformedInputNeverBecomesAnAction() throws {
+ let invalid: [[String: Any]] = [
+ ["type": "click", "x": true, "y": 1],
+ ["type": "click", "x": Double.nan, "y": 1],
+ ["type": "click", "x": -1, "y": 1],
+ ["type": "click", "x": 1, "y": 1, "strict": false],
+ ["type": "click", "x": 1],
+ ["type": "scroll", "x": 1, "y": 1, "delta_x": 1.5, "delta_y": 1],
+ ["type": "key", "key": "command+space"],
+ ["type": "key", "key": "paste"],
+ ["type": "type", "text": "hello\0world"],
+ ["type": "type", "text": String(repeating: "x", count: 8001)],
+ ["type": "drag", "path": [["x": 1, "y": 1]]],
+ ["type": "shell", "text": "ignored"],
+ ]
+ for candidate in invalid { #expect(throws: (any Error).self) { try Action(candidate) } }
+ _ = try Action(["type": "set_value", "ref": "e1", "text": ""])
+ _ = try Action(["type": "type", "text": "Hello 👋🏽 café 日本語"])
+ #expect(throws: (any Error).self) { try Arguments(values: ["pid": true]).integer("pid", min: 1, max: 999) }
+ #expect(throws: (any Error).self) { try Arguments(values: ["include_image": "false"]).bool("include_image", default: true) }
+}
- @MainActor
- func testNoSessionMeansNoContentOrInput() async throws {
- let runtime = SessionRuntime()
- let calls: [(String, [String: Any])] = [
- ("computer_observe", ["session_id": "invented"]),
- ("computer_act", ["session_id": "invented", "observation_id": "invented", "request_id": "one", "action": ["type": "click", "x": 10, "y": 20]]),
- ("computer_session_status", ["session_id": "invented"]),
- ("computer_close_session", ["session_id": "invented"]),
- ]
- for (name, args) in calls {
- do { _ = try await runtime.call(name, args); XCTFail("Unapproved access succeeded") }
- catch let error as UseError { XCTAssertEqual(error.code, "session_unavailable") }
- }
+@Test func observationCannotCrossWindowGenerationTimeOrImageBoundary() throws {
+ let frame = CGRect(x: -1440, y: 200, width: 1200, height: 800)
+ let observation = ObservationLease(id: "observed", createdAt: 100, generation: 7, frame: frame, imageWidth: 600, imageHeight: 400)
+ try observation.validate(id: "observed", generation: 7, frame: frame, now: 101)
+ #expect(throws: (any Error).self) { try observation.validate(id: "other-session", generation: 7, frame: frame, now: 101) }
+ #expect(throws: (any Error).self) { try observation.validate(id: "observed", generation: 8, frame: frame, now: 101) }
+ #expect(throws: (any Error).self) { try observation.validate(id: "observed", generation: 7, frame: frame, now: 116) }
+ #expect(throws: (any Error).self) { try observation.validate(id: "observed", generation: 7, frame: frame.offsetBy(dx: 1, dy: 0), now: 101) }
+ #expect(try observation.screenPoint(CGPoint(x: 300, y: 200)) == CGPoint(x: -840, y: 600))
+ #expect(throws: (any Error).self) { try observation.screenPoint(CGPoint(x: 600, y: 0)) }
+ #expect(throws: (any Error).self) { try observation.screenPoint(CGPoint(x: 0, y: 400)) }
+ #expect(throws: (any Error).self) { try observation.screenPoint(CGPoint(x: Double.infinity, y: 0)) }
+}
+
+@MainActor
+@Test func noSessionMeansNoContentOrInput() async throws {
+ let runtime = SessionRuntime()
+ let calls: [(String, [String: Any])] = [
+ ("computer_observe", ["session_id": "invented"]),
+ ("computer_act", ["session_id": "invented", "observation_id": "invented", "request_id": "one", "action": ["type": "click", "x": 10, "y": 20]]),
+ ("computer_session_status", ["session_id": "invented"]),
+ ("computer_close_session", ["session_id": "invented"]),
+ ]
+ for (name, args) in calls {
+ do { _ = try await runtime.call(name, args); Issue.record("Unapproved access succeeded") }
+ catch let error as UseError { #expect(error.code == "session_unavailable") }
}
+}
- @MainActor
- func testLegacyEscapeHatchesAreNotTools() async throws {
- let runtime = SessionRuntime()
- for name in ["snapshot", "set_strict_mode", "cua_screenshot", "cua_click", "clipboard_read", "clipboard_write", "open_url", "launch_app"] {
- do { _ = try await runtime.call(name, [:]); XCTFail("Legacy tool succeeded") }
- catch let error as UseError { XCTAssertEqual(error.code, "unknown_tool") }
- }
- let names = Set(MCPServer.schemas().compactMap { $0["name"] as? String })
- XCTAssertEqual(names, ["computer_discover", "computer_open_session", "computer_observe", "computer_act", "computer_session_status", "computer_close_session"])
+@MainActor
+@Test func legacyEscapeHatchesAreNotTools() async throws {
+ let runtime = SessionRuntime()
+ for name in ["snapshot", "set_strict_mode", "cua_screenshot", "cua_click", "clipboard_read", "clipboard_write", "open_url", "launch_app"] {
+ do { _ = try await runtime.call(name, [:]); Issue.record("Legacy tool succeeded") }
+ catch let error as UseError { #expect(error.code == "unknown_tool") }
}
+ let names = Set(MCPServer.schemas().compactMap { $0["name"] as? String })
+ #expect(names == ["computer_discover", "computer_open_session", "computer_observe", "computer_act", "computer_session_status", "computer_close_session"])
}
diff --git a/packages/computer-use/native/Tests/ComputerUseTests/WorkerTests.swift b/packages/computer-use/native/Tests/ComputerUseTests/WorkerTests.swift
new file mode 100644
index 00000000..dfa7df0c
--- /dev/null
+++ b/packages/computer-use/native/Tests/ComputerUseTests/WorkerTests.swift
@@ -0,0 +1,105 @@
+import Foundation
+import CoreGraphics
+import Testing
+@testable import ComputerUse
+
+// The native worker's pure parts: contract v2 key names and modifiers, the
+// coordinate spaces, the stdout framing and the synthetic-input rule.
+
+@Test func everyContractKeyMapsToAKeyCode() {
+ let named = ["Enter", "Tab", "Escape", "Backspace", "Delete", "Insert", "Space", "Left", "Up", "Right", "Down",
+ "Home", "End", "PageUp", "PageDown", "CapsLock", "ContextMenu", "PrintScreen"]
+ for key in named { #expect(WorkerKeys.resolve(key) != nil, "\(key)") }
+ for n in 1...20 { #expect(WorkerKeys.resolve("F\(n)") != nil) }
+ for key in ["F21", "F24", "F0", "Fx", "enter", "ab", ""] { #expect(WorkerKeys.resolve(key) == nil, "\(key)") }
+ for char in "abcdefghijklmnopqrstuvwxyz0123456789`-=[]\\;',./" { #expect(WorkerKeys.resolve(String(char))?.shift == false, "\(char)") }
+ for char in "~!@#$%^&*()_+{}|:\"<>?" { #expect(WorkerKeys.resolve(String(char))?.shift == true, "\(char)") }
+ #expect(WorkerKeys.resolve("a") == .init(code: 0, shift: false, text: "a"))
+ #expect(WorkerKeys.resolve("!") == .init(code: 18, shift: true, text: "!"))
+ #expect(WorkerKeys.resolve("Space")?.text == " ")
+ #expect(WorkerKeys.resolve("Enter")?.text == nil)
+}
+
+@Test func modifiersAndBlockedChords() {
+ #expect(WorkerKeys.flags([]) == [])
+ #expect(WorkerKeys.flags(["ctrl", "shift", "alt", "meta"]) == [.maskControl, .maskShift, .maskAlternate, .maskCommand])
+ #expect(WorkerKeys.flags(["cmd"]) == nil)
+ #expect(WorkerKeys.blocked("Tab", ["meta"]))
+ #expect(WorkerKeys.blocked("Space", ["meta"]))
+ #expect(WorkerKeys.blocked("Escape", ["meta", "alt"]))
+ #expect(WorkerKeys.blocked("q", ["ctrl", "meta"]))
+ #expect(WorkerKeys.blocked("q", ["shift", "meta"]))
+ #expect(!WorkerKeys.blocked("s", ["meta"]))
+ #expect(!WorkerKeys.blocked("q", ["meta"])) // quit asks the person first (policy.mjs), it is not refused here
+ #expect(!WorkerKeys.blocked("Enter", []))
+}
+
+@Test func screenRectsMapToCapturePixels() throws {
+ // A 600×400 pt window at (-1440, 200) captured at 2× = 1200×800 px.
+ let bounds = CGRect(x: -1440, y: 200, width: 600, height: 400)
+ let field = CGRect(x: -1400, y: 250, width: 100, height: 20)
+ let px = try #require(WorkerGeometry.pixels(of: field, bounds: bounds, width: 1200, height: 800))
+ #expect(px == CGRect(x: 80, y: 100, width: 200, height: 40))
+ let padded = try #require(WorkerGeometry.pixels(of: field, bounds: bounds, width: 1200, height: 800, pad: 2))
+ #expect(padded == CGRect(x: 78, y: 98, width: 204, height: 44))
+ // Partly outside: clipped to the window and the image.
+ let edge = try #require(WorkerGeometry.pixels(of: CGRect(x: -1450, y: 190, width: 20, height: 20), bounds: bounds, width: 1200, height: 800, pad: 4))
+ #expect(edge == CGRect(x: 0, y: 0, width: 24, height: 24))
+ #expect(WorkerGeometry.pixels(of: CGRect(x: 0, y: 0, width: 5, height: 5), bounds: bounds, width: 1200, height: 800) == nil)
+}
+
+@Test func visionBoxesAreFlippedToTopLeftPixels() {
+ let box = WorkerGeometry.pixels(ofNormalized: CGRect(x: 0.25, y: 0.5, width: 0.5, height: 0.25), width: 1000, height: 800)
+ #expect(box == CGRect(x: 250, y: 200, width: 500, height: 200))
+}
+
+@Test func captureSizeIsNativeButCapped() {
+ #expect(WorkerGeometry.captureSize(points: CGSize(width: 1440, height: 900), scale: 2, maxLongSide: 3840) == (2880, 1800))
+ #expect(WorkerGeometry.captureSize(points: CGSize(width: 2560, height: 1440), scale: 2, maxLongSide: 3840) == (3840, 2160))
+ #expect(WorkerGeometry.captureSize(points: CGSize(width: 800, height: 600), scale: 1, maxLongSide: 3840) == (800, 600))
+}
+
+@Test func wordsKeepTheirRanges() {
+ let text = "password: hunter2 x"
+ let words = WorkerGeometry.words(text)
+ #expect(words.map(\.0) == ["password:", "hunter2", "x"])
+ #expect(String(text[words[1].1]) == "hunter2")
+ #expect(WorkerGeometry.words(" ").isEmpty)
+}
+
+@Test func syntheticInputRule() {
+ let hid = Int64(CGEventSourceStateID.hidSystemState.rawValue)
+ #expect(!WorkerInputSource.isSynthetic(sourcePid: 0, stateID: hid, sourceIsWindowServer: false))
+ #expect(!WorkerInputSource.isSynthetic(sourcePid: 123, stateID: hid, sourceIsWindowServer: true))
+ #expect(WorkerInputSource.isSynthetic(sourcePid: 4567, stateID: hid, sourceIsWindowServer: false))
+ #expect(WorkerInputSource.isSynthetic(sourcePid: 0, stateID: Int64(CGEventSourceStateID.combinedSessionState.rawValue), sourceIsWindowServer: false))
+ #expect(WorkerInputSource.isSynthetic(sourcePid: 0, stateID: 987_654, sourceIsWindowServer: false))
+}
+
+@Test func framesCarryTheirBinaryLength() throws {
+ let pixels = Data([1, 2, 3, 4, 5, 6, 7, 8])
+ let data = try #require(WorkerFrame.encode(["id": 3, "result": ["width": 2, "height": 1]], binary: pixels))
+ let newline = try #require(data.firstIndex(of: 10))
+ let header = try #require(try JSONSerialization.jsonObject(with: data[..<newline]) as? [String: Any])
+ let result = try #require(header["result"] as? [String: Any])
+ #expect(result["bytes"] as? Int == 8)
+ #expect(result["width"] as? Int == 2)
+ #expect(Data(data[(newline + 1)...]) == pixels)
+ let plain = try #require(WorkerFrame.encode(["event": "person-input", "generation": 2]))
+ #expect(plain.last == 10)
+ #expect(plain.filter { $0 == 10 }.count == 1)
+}
+
+@MainActor
+@Test func workerParametersAreStrict() throws {
+ #expect(NativeWorker.int(NSNumber(value: 3)) == 3)
+ #expect(NativeWorker.int(NSNumber(value: 3.5)) == nil)
+ #expect(NativeWorker.int(kCFBooleanTrue) == nil)
+ #expect(NativeWorker.number("1") == nil)
+ #expect(try NativeWorker.windowID(NSNumber(value: 42)) == 42)
+ #expect(try NativeWorker.windowID("42") == 42)
+ #expect(throws: (any Error).self) { try NativeWorker.windowID(NSNumber(value: 0)) }
+ #expect(throws: (any Error).self) { try NativeWorker.windowID("-1") }
+ #expect(!NativeWorker.releaseOnly("not json"))
+ #expect(!NativeWorker.releaseOnly(#"{"pid":0,"window":1,"x":0,"y":0,"frame":[0,0,10,10]}"#))
+}
diff --git a/packages/types/src/desktop-ipc.ts b/packages/types/src/desktop-ipc.ts
index eb4bc58d..89fe8488 100644
--- a/packages/types/src/desktop-ipc.ts
+++ b/packages/types/src/desktop-ipc.ts
@@ -876,6 +876,79 @@ export type RunningAppsResult = {
apps: string[];
};
+/** Computer use (contract v2): the app identity the host verified (exe path / bundle id, pid). */
+export type ComputerUseAppRef = { app_id: string; app_name: string; bundle_or_exe: string; pid: number };
+
+/** One computer use session as the host sees it (apps/server/src/computer-use/shared/portable-host.mjs). */
+export type ComputerUseSessionState = {
+ id: string;
+ phase: "approval" | "working" | "paused" | "ended";
+ mode: "observe" | "control";
+ purpose: string;
+ status: string;
+ app: ComputerUseAppRef;
+ window: { id: string; title: string };
+ windows: Array<{ id: string; title: string }>;
+ approval: "once" | "chat" | null;
+ actions: number;
+ actions_left: number;
+ seconds_left: number;
+ can_continue: boolean;
+ log: Array<{ at: string; action: string; intent: string; ok: boolean; code: string | null }>;
+ frame: { sha: string; width: number; height: number; mime: string; data_url: string; at: string } | null;
+ chat: string | null;
+ hidden?: boolean;
+ window_bounds?: { x: number; y: number; width: number; height: number };
+};
+
+/** A question only the person answers, in the main window (never through the engine). */
+export type ComputerUseRequest =
+ | {
+ kind: "open";
+ request_id: string;
+ session_id: string;
+ app: ComputerUseAppRef;
+ /** The app's icon (PNG data URL) when the OS gives one. */
+ icon: string | null;
+ windows: Array<{ id: string; title: string }>;
+ window_id: string;
+ mode: "observe" | "control";
+ purpose: string;
+ os: string;
+ seconds: number;
+ chat_seconds: number;
+ chat: string | null;
+ }
+ | {
+ kind: "act";
+ request_id: string;
+ session_id: string;
+ app: ComputerUseAppRef;
+ icon: string | null;
+ window_title: string;
+ purpose: string;
+ intent: string;
+ action: Record<string, unknown>;
+ reasons: Array<{ code: string; detail?: string; text: string }>;
+ /** Points of the action, in pixels of `frame`. */
+ points: Array<{ x: number; y: number }>;
+ frame: { width: number; height: number; sha256: string; data_url: string };
+ chat: string | null;
+ };
+
+export type ComputerUseAnswer = { request_id: string; decision: "chat" | "once" | "deny" | "allow"; window_id?: string };
+export type ComputerUseControlAction = "stop" | "takeover" | "continue" | "hide" | "show";
+
+export type ComputerUseStatus = {
+ enabled: boolean;
+ supported: boolean;
+ /** Why it is off or unavailable (people-facing), when it is. */
+ message: string | null;
+ mode: "careful" | "normal" | "auto";
+ sessions: ComputerUseSessionState[];
+ requests: ComputerUseRequest[];
+};
+
// ---------------------------------------------------------------------------
// The command map
// ---------------------------------------------------------------------------
@@ -962,8 +1035,9 @@ export type DesktopCommandMap = {
getOmniRushUiMcpCommand: { args: []; result: string[] };
getComputerUseMcpCommand: { args: []; result: string[] };
getComputerUseMcpEnvironment: { args: []; result: Record<string, string> };
- getComputerUseState: { args: []; result: unknown };
- computerUseAction: { args: [value: { connectionId: string; id: string; action: string; windowId?: number }]; result: void };
+ getComputerUseState: { args: []; result: ComputerUseStatus };
+ computerUseAnswer: { args: [answer: ComputerUseAnswer]; result: void };
+ computerUseControl: { args: [value: { session_id: string; action: ComputerUseControlAction }]; result: void };
getOmniRushUiMcpEnvironment: { args: []; result: Record<string, string> };
// Computer use