A desktop coding agent with free access to frontier models.
TypeScript
fix: answers need a press on the button
ogsamratdc898cf00d3f
diff --git a/apps/app/src/react-app/domains/session/surface/computer-use/live-panel.tsx b/apps/app/src/react-app/domains/session/surface/computer-use/live-panel.tsxindex a2635992..27a13a1d 100644--- a/apps/app/src/react-app/domains/session/surface/computer-use/live-panel.tsx+++ b/apps/app/src/react-app/domains/session/surface/computer-use/live-panel.tsx@@ -7,7 +7,7 @@ import { stripRemoteMethodErrorPrefix } from "@/react-app/domains/settings/stateimport { formatMinutesLeft } from "./describe";import { ComputerUseAppIdentity, ComputerUseFrame } from "./request-card";-import { controlComputerUse } from "./status-store";+import { buttonRect, controlComputerUse } from "./status-store";/** A running (or paused) computer use session: what it sees, what it did, and Stop. */export function ComputerUseLivePanel(props: { session: ComputerUseSessionState }) {@@ -17,10 +17,10 @@ export function ComputerUseLivePanel(props: { session: ComputerUseSessionState }const [error, setError] = useState<string | null>(null);const paused = session.phase === "paused";- const control = (action: ComputerUseControlAction) => {+ const control = (action: ComputerUseControlAction, event?: { currentTarget: EventTarget | null }) => {setBusy(true);setError(null);- controlComputerUse(session.id, action)+ controlComputerUse(session.id, action, buttonRect(event?.currentTarget ?? null)).catch((reason: unknown) => setError(reason instanceof Error ? stripRemoteMethodErrorPrefix(reason.message) : "That did not work. Try again.")).finally(() => setBusy(false));};@@ -28,18 +28,18 @@ export function ComputerUseLivePanel(props: { session: ComputerUseSessionState }const buttons = (<div className="flex shrink-0 flex-wrap items-center gap-2">{paused && session.can_continue ? (- <Button type="button" variant="outline" size="xs" disabled={busy} onClick={() => control("continue")} data-computer-use-control="continue">+ <Button type="button" variant="outline" size="xs" disabled={busy} onClick={(event) => control("continue", event)} data-computer-use-control="continue"><Play data-icon="inline-start" />Continue</Button>) : null}{!paused && session.mode === "control" ? (- <Button type="button" variant="outline" size="xs" disabled={busy} onClick={() => control("takeover")} data-computer-use-control="takeover">+ <Button type="button" variant="outline" size="xs" disabled={busy} onClick={(event) => control("takeover", event)} data-computer-use-control="takeover"><Hand data-icon="inline-start" />Take over</Button>) : null}- <Button type="button" variant="destructive" size="xs" disabled={busy} onClick={() => control("stop")} data-computer-use-control="stop">+ <Button type="button" variant="destructive" size="xs" disabled={busy} onClick={(event) => control("stop", event)} data-computer-use-control="stop"><Square data-icon="inline-start" />Stop</Button>diff --git a/apps/app/src/react-app/domains/session/surface/computer-use/request-card.tsx b/apps/app/src/react-app/domains/session/surface/computer-use/request-card.tsxindex 02746631..9db5a30b 100644--- a/apps/app/src/react-app/domains/session/surface/computer-use/request-card.tsx+++ b/apps/app/src/react-app/domains/session/surface/computer-use/request-card.tsx@@ -6,7 +6,7 @@ import { Button } from "@/components/ui/button";import { stripRemoteMethodErrorPrefix } from "@/react-app/domains/settings/state/electron-updater-state";import { describeComputerUseAction, scalePoints, typedText, type ComputerUsePoint } from "./describe";-import { answerComputerUse } from "./status-store";+import { answerComputerUse, buttonRect } from "./status-store";const APPROVE_WITH_OWN_INPUT = "Use your own mouse or keyboard to approve.";@@ -88,10 +88,11 @@ export function ComputerUseRequestCard(props: { request: ComputerUseRequest }) {const [busy, setBusy] = useState(false);const [error, setError] = useState<string | null>(null);- const answer = (decision: ComputerUseAnswer["decision"]) => {+ const answer = (decision: ComputerUseAnswer["decision"], event?: { currentTarget: EventTarget | null }) => {setBusy(true);setError(null);- answerComputerUse({ request_id: request.request_id, decision, ...(request.kind === "open" ? { window_id: windowId } : {}) })+ const rect = buttonRect(event?.currentTarget ?? null);+ answerComputerUse({ request_id: request.request_id, decision, ...(request.kind === "open" ? { window_id: windowId } : {}), ...(rect ? { rect } : {}) }).catch((reason: unknown) => {const message = reason instanceof Error ? stripRemoteMethodErrorPrefix(reason.message) : "";setError(message || APPROVE_WITH_OWN_INPUT);@@ -140,18 +141,18 @@ export function ComputerUseRequestCard(props: { request: ComputerUseRequest }) {</p>{errorLine}<div className="flex flex-wrap items-center gap-2">- <Button type="button" size="sm" disabled={busy} onClick={() => answer("chat")} data-computer-use-answer="chat" className="h-auto py-1.5">+ <Button type="button" size="sm" disabled={busy} onClick={(event) => answer("chat", event)} data-computer-use-answer="chat" className="h-auto py-1.5"><Check data-icon="inline-start" /><span className="flex flex-col items-start leading-4"><span>Allow for this chat</span><span className="text-[10px] font-normal opacity-80">Same app, up to {Math.round(request.chat_seconds / 60)} minutes</span></span></Button>- <Button type="button" variant="outline" size="sm" disabled={busy} onClick={() => answer("once")} data-computer-use-answer="once">+ <Button type="button" variant="outline" size="sm" disabled={busy} onClick={(event) => answer("once", event)} data-computer-use-answer="once"><Clock3 data-icon="inline-start" />Allow once</Button>- <Button type="button" variant="destructive" size="sm" disabled={busy} onClick={() => answer("deny")} data-computer-use-answer="deny">+ <Button type="button" variant="destructive" size="sm" disabled={busy} onClick={(event) => answer("deny", event)} data-computer-use-answer="deny"><XCircle data-icon="inline-start" />Deny</Button>@@ -185,11 +186,11 @@ export function ComputerUseRequestCard(props: { request: ComputerUseRequest }) {<ComputerUseFrame frame={request.frame} points={request.points} alt={`${request.app.app_name} window`} />{errorLine}<div className="flex flex-wrap items-center gap-2">- <Button type="button" size="sm" disabled={busy} onClick={() => answer("allow")} data-computer-use-answer="allow">+ <Button type="button" size="sm" disabled={busy} onClick={(event) => answer("allow", event)} data-computer-use-answer="allow"><Check data-icon="inline-start" />Allow</Button>- <Button type="button" variant="destructive" size="sm" disabled={busy} onClick={() => answer("deny")} data-computer-use-answer="deny">+ <Button type="button" variant="destructive" size="sm" disabled={busy} onClick={(event) => answer("deny", event)} data-computer-use-answer="deny"><XCircle data-icon="inline-start" />Deny</Button>diff --git a/apps/app/src/react-app/domains/session/surface/computer-use/status-store.ts b/apps/app/src/react-app/domains/session/surface/computer-use/status-store.tsindex ec4fdb87..1667d618 100644--- a/apps/app/src/react-app/domains/session/surface/computer-use/status-store.ts+++ b/apps/app/src/react-app/domains/session/surface/computer-use/status-store.ts@@ -5,7 +5,14 @@import { useEffect, useSyncExternalStore } from "react";import { create } from "zustand";-import type { ComputerUseAnswer, ComputerUseControlAction, ComputerUseStatus } from "@omnirush/types/desktop-ipc";+import type { ComputerUseAnswer, ComputerUseButtonRect, ComputerUseControlAction, ComputerUseStatus } from "@omnirush/types/desktop-ipc";++/** The pressed button's box, for main's check that the person pressed it themselves. */+export function buttonRect(target: EventTarget | null): ComputerUseButtonRect | undefined {+ if (!(target instanceof Element)) return undefined;+ const r = target.getBoundingClientRect();+ return { x: r.x, y: r.y, width: r.width, height: r.height };+}let current: ComputerUseStatus | null = null;const listeners = new Set<() => void>();@@ -81,8 +88,8 @@ export async function answerComputerUse(answer: ComputerUseAnswer) {}/** Stop, take over or continue a session. */-export async function controlComputerUse(session_id: string, action: ComputerUseControlAction) {+export async function controlComputerUse(session_id: string, action: ComputerUseControlAction, rect?: ComputerUseButtonRect) {const invoke = window.__OMNIRUSH_ELECTRON__?.invokeDesktop;if (!invoke) throw new Error("Computer use needs the desktop app.");- await invoke("computerUseControl", { session_id, action });+ await invoke("computerUseControl", { session_id, action, ...(rect ? { rect } : {}) });}diff --git a/apps/desktop/electron/computer-use.mjs b/apps/desktop/electron/computer-use.mjsindex 31ec1902..d3cb6f39 100644--- a/apps/desktop/electron/computer-use.mjs+++ b/apps/desktop/electron/computer-use.mjs@@ -18,7 +18,7 @@ import { createRequire } from "node:module";import path from "node:path";import { fileURLToPath, pathToFileURL } from "node:url";-import { app } from "electron";+import { app, screen } from "electron";import { testAnswersAllowed } from "./computer-use-answers.mjs";import { createComputerUseHost } from "./computer-use-host.mjs";import { createComputerPreview } from "./computer-use-preview.mjs";@@ -38,10 +38,25 @@ const SYNTHETIC_MESSAGE = "Use your own mouse or keyboard to approve.";// remote-debugging clicks, in builds run from source only (the e2e rig).// Packaged apps ignore it (computer-use-answers.mjs).const testAnswers = testAnswersAllowed(process.env, app.isPackaged, (message) => console.warn(`[computer-use] ${message}`));-async function personAnswered() {+// The pressed button in screen pixels (the worker's press coordinates): content bounds + CSS box × zoom.+function screenRectOf(rect) {+ const window = deps?.getWindow?.();+ if (!window || window.isDestroyed() || !rect || ![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite)) return null;+ const content = window.getContentBounds(), zoom = window.webContents.getZoomFactor();+ const dip = { x: Math.round(content.x + rect.x * zoom), y: Math.round(content.y + rect.y * zoom), width: Math.round(rect.width * zoom), height: Math.round(rect.height * zoom) };+ if (process.platform === "win32") return screen.dipToScreenRect(window, dip);+ if (process.platform === "linux") {+ const factor = screen.getDisplayMatching(dip).scaleFactor || 1;+ return { x: dip.x * factor, y: dip.y * factor, width: dip.width * factor, height: dip.height * factor };+ }+ return dip; // macOS: points, as the helper reports presses+}++// A physical press within 1.5 s, in this app's window, on the pressed button (or Enter/Space on it).+async function personAnswered(rect) {if (!host) return false;if (await host.syntheticInput()) return false;- return testAnswers || host.physicalInput(5000, { foregroundPid: process.pid });+ return testAnswers || host.physicalInput(1500, { foregroundPid: process.pid, rect: screenRectOf(rect) });}let deps = null;@@ -259,14 +274,14 @@ async function computerUseAnswer(answer) {const entry = pending.get(answer?.request_id);if (!entry) throw new Error("This request has ended.");const allowing = answer.decision !== "deny";- if (allowing && !(await personAnswered())) throw new Error(SYNTHETIC_MESSAGE);+ if (allowing && !(await personAnswered(answer.rect))) throw new Error(SYNTHETIC_MESSAGE);const decision = entry.request.kind === "open" ? (["chat", "once", "deny"].includes(answer.decision) ? answer.decision : "deny") : (answer.decision === "allow" ? "allow" : "deny");entry.resolve({ decision, ...(typeof answer.window_id === "string" ? { window_id: answer.window_id } : {}) });}async function computerUseControl(value) {if (!host) throw new Error("Computer use is not running.");- if (value?.action === "continue" && !(await personAnswered())) throw new Error(SYNTHETIC_MESSAGE);+ if (value?.action === "continue" && !(await personAnswered(value.rect))) throw new Error(SYNTHETIC_MESSAGE);await host.control({ session_id: value?.session_id, action: value?.action });pushStatus();}diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.tsindex 1b2f5391..600a77d8 100644--- a/apps/server/src/computer-use/portable-host.test.ts+++ b/apps/server/src/computer-use/portable-host.test.ts@@ -24,7 +24,7 @@ function fakeNative() {const listeners = new Set<(v: Record<string, unknown>) => void>();const calls: Array<{ method: string; params: Record<string, unknown> }> = [];let generation = 0;- const state: { occluded: boolean; synthetic: boolean; retitle: ((title: string) => string) | null; texts: Array<{ x: number; y: number; width: number; height: number; text: string }> | null } = { occluded: false, synthetic: false, retitle: null, texts: null };+ const state: { occluded: boolean; synthetic: boolean; retitle: ((title: string) => string) | null; texts: Array<{ x: number; y: number; width: number; height: number; text: string }> | null; pressAge?: number; pressKind?: string; pressKey?: string; pressAt?: { x: number; y: number }; pressPid?: number } = { occluded: false, synthetic: false, retitle: null, texts: null };const frame = (w: Window): Frame => {const width = Math.min(w.bounds.width, 640), height = Math.round((width * w.bounds.height) / w.bounds.width);const data = new Uint8Array(width * height * 4).fill(shade);@@ -52,7 +52,7 @@ function fakeNative() {if (method === "state") return { generation, available: true };if (method === "start") return { generation };if (method === "hit") return { inside: (params.points as unknown[]).map(() => !state.occluded) };- if (method === "input-source") return { synthetic: state.synthetic, at: Date.now(), kind: "pointer" };+ if (method === "input-source") return { synthetic: state.synthetic, at: Date.now() - (state.pressAge ?? 0), kind: state.pressKind ?? "pointer", key: state.pressKey ?? null, x: state.pressAt?.x ?? null, y: state.pressAt?.y ?? null, foreground_pid: state.pressPid ?? null };if (method === "act") {if (params.generation !== generation) throw Object.assign(new Error("Your input paused computer use."), { code: "paused_by_person" });if (state.retitle) windows[0]!.title = state.retitle(windows[0]!.title);@@ -369,6 +369,27 @@ describe("portable host: titles that follow the agent's own typing, and a worker});});+describe("portable host: the person's own press answers a question", () => {+ test("within 1.5 s, in the app's window, on the pressed button or with Enter/Space", async () => {+ const r = rig("normal");+ const host = await r.hostPromise;+ const rect = { x: 100, y: 200, width: 80, height: 30 };+ r.native.state.pressAt = { x: 120, y: 210 }; r.native.state.pressPid = 4242;+ expect(await host.physicalInput(1500, { foregroundPid: 4242, rect })).toBe(true);+ r.native.state.pressAt = { x: 400, y: 210 };+ expect(await host.physicalInput(1500, { foregroundPid: 4242, rect })).toBe(false);+ r.native.state.pressAt = { x: 120, y: 210 };+ expect(await host.physicalInput(1500, { foregroundPid: 999, rect })).toBe(false);+ r.native.state.pressAge = 2000;+ expect(await host.physicalInput(1500, { foregroundPid: 4242, rect })).toBe(false);+ r.native.state.pressAge = 0; r.native.state.pressKind = "key"; r.native.state.pressKey = "Enter";+ expect(await host.physicalInput(1500, { foregroundPid: 4242, rect })).toBe(true);+ r.native.state.pressKey = "y";+ expect(await host.physicalInput(1500, { foregroundPid: 4242, rect })).toBe(false);+ await host.close();+ });+});+describe("portable host: the control under the point from the worker (Windows MSAA)", () => {test("a named control decides (risky target, protected field); no name falls back to target_unknown", async () => {const r = rig("normal");diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256index 940d1332..f96eef5b 100644--- a/apps/server/src/computer-use/shared/PARITY.sha256+++ b/apps/server/src/computer-use/shared/PARITY.sha256@@ -8,6 +8,6 @@ a96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs5eac31868aedd3e1d1e71b445f9a55d01316da7397f27f6db11472453bee4588 image.d.mtse47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjseabdf78c56733577d05aad490d9d7d5ece6c22e9b3ea9e6e9803d1a4a546fdd4 media.d.mts-8543b410d31a33e700f4f6dbde3547ad0caa92421af387fe3e772a7444edfd60 native-worker.mjs-7a55a915d4d725a4e992e7ef6191ba0ad1c851cdb5ee712ef79e0f7981a6a1c4 portable-host.mjs-2ed7be072df62d50e1b569e75b695d18ae0d0de6dabe5375bc494b859cf78710 portable-host.d.mts+a6a6596c75df17c0aac6d7c740ae9661cae2b8899d12e20b5c2f37987cf9d4be native-worker.mjs+110b5f3aadaaec96b523fde230fa513d0fd75f2f7634b7336ac5b4ba975f18f4 portable-host.mjs+641be80cebb04d353031392d8f53f45ead1e1beaf83f1e67d24945f0eda330bf portable-host.d.mtsdiff --git a/apps/server/src/computer-use/shared/native-worker.mjs b/apps/server/src/computer-use/shared/native-worker.mjsindex 31db5551..bec6109e 100644--- a/apps/server/src/computer-use/shared/native-worker.mjs+++ b/apps/server/src/computer-use/shared/native-worker.mjs@@ -32,12 +32,12 @@ const presses = [];function notePress(entry) {Object.assign(lastInput, { synthetic: entry.synthetic, at: entry.at, kind: entry.kind });if (entry.synthetic) return;- presses.push({ at: entry.at, kind: entry.kind, key: entry.key ?? null, foreground_pid: entry.foreground_pid ?? null });+ presses.push({ at: entry.at, kind: entry.kind, key: entry.key ?? null, foreground_pid: entry.foreground_pid ?? null, x: entry.x ?? null, y: entry.y ?? null });if (presses.length > 16) presses.shift();}function inputSource() {const physical = presses.at(-1) ?? null;- return { ...lastInput, key: physical?.key ?? null, foreground_pid: physical?.foreground_pid ?? null, physical_at: physical?.at ?? 0, presses: presses.slice() };+ return { ...lastInput, key: physical?.key ?? null, foreground_pid: physical?.foreground_pid ?? null, x: physical?.x ?? null, y: physical?.y ?? null, physical_at: physical?.at ?? 0, presses: presses.slice() };}const changed = () => { generation += 1; post({ event: "person-input", generation }); };const fail = (code, message) => { throw Object.assign(new Error(message), { code }); };@@ -169,7 +169,8 @@ function windows() {if (down) {const fg = [0];try { pidOf(foreground(), fg); } catch { /* unknown */ }- notePress({ synthetic: (value.flags & injectedFlag) !== 0, at: Date.now(), kind: type === hookKey ? "key" : "pointer", key: type === hookKey ? keyFromVirtualKey(Number(value.vkCode)) : null, foreground_pid: fg[0] || null });+ const at = type === hookMouse ? { x: value.pt.x, y: value.pt.y } : {};+ notePress({ synthetic: (value.flags & injectedFlag) !== 0, at: Date.now(), kind: type === hookKey ? "key" : "pointer", key: type === hookKey ? keyFromVirtualKey(Number(value.vkCode)) : null, foreground_pid: fg[0] || null, ...at });}if (interrupting) { changed(); release(); }}@@ -570,12 +571,13 @@ function x11() {if (ours) continue;if (cookie.evtype === 13 || cookie.evtype === 15) {const synthetic = testDevices.has(raw.sourceid);- let key = null, fg = null;+ let key = null, fg = null, where = {};if (!synthetic) {+ try { if (cookie.evtype === 15) { const r = [0], c = [0], rx = [0], ry = [0], wx = [0], wy = [0], mk = [0]; if (queryPointer(display, root, r, c, rx, ry, wx, wy, mk)) where = { x: rx[0], y: ry[0] }; } } catch { /* unknown */ }try { if (cookie.evtype === 13) key = keyFromKeysymName(keysymName(keycodeKeysym(display, raw.detail, 0, 0))); } catch { /* unknown */ }try { fg = Number(prop(activeWindow(), "_NET_WM_PID")[0] ?? 0) || null; } catch { /* unknown */ }}- notePress({ synthetic, at: Date.now(), kind: cookie.evtype === 13 ? "key" : "pointer", key, foreground_pid: fg });+ notePress({ synthetic, at: Date.now(), kind: cookie.evtype === 13 ? "key" : "pointer", key, foreground_pid: fg, ...where });}if (interrupting) changed();}diff --git a/apps/server/src/computer-use/shared/portable-host.d.mts b/apps/server/src/computer-use/shared/portable-host.d.mtsindex bfb78b6d..a569c864 100644--- a/apps/server/src/computer-use/shared/portable-host.d.mts+++ b/apps/server/src/computer-use/shared/portable-host.d.mts@@ -80,7 +80,8 @@ export type PortableHost = {stopAll(by?: SessionEvent["by"], event?: SessionEvent["event"]): Promise<void>;syntheticInput(withinMs?: number): Promise<boolean>;/** A physical key or button press within `withinMs` (not injected, not a remote-debugging click). */- physicalInput(withinMs?: number, options?: { foregroundPid?: number | null }): Promise<boolean>;+ /** Default 1.5 s; a pointer press must be inside `rect` (screen pixels) when given; a key press must be one of `keys` (default Enter, Space). */+ physicalInput(withinMs?: number, options?: { foregroundPid?: number | null; rect?: { x: number; y: number; width: number; height: number } | null; keys?: string[] }): Promise<boolean>;watchInput(): Promise<void>;state(): HostSessionState[];close(): Promise<void>;diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjsindex 674ddd76..7d8f8fd8 100644--- a/apps/server/src/computer-use/shared/portable-host.mjs+++ b/apps/server/src/computer-use/shared/portable-host.mjs@@ -656,11 +656,18 @@ export async function createPortableHost({ native: firstNative, nativeFactory =* Whether a physical key or button press (not injected by a program, not a remote-debugging* click that makes no OS input) came within `withinMs`: the person's own answer.*/- async function physicalInput(withinMs = 5000, { foregroundPid = null } = {}) {+ async function physicalInput(withinMs = 1500, { foregroundPid = null, rect = null, keys = ["Enter", "Space"] } = {}) {const value = await native.call("input-source").catch(() => null);if (!value || !value.at || value.synthetic || Date.now() - value.at >= withinMs) return false;// In the answering app's window (when the worker knows which app was in front).- return !(foregroundPid && Number.isInteger(value.foreground_pid) && value.foreground_pid !== foregroundPid);+ if (foregroundPid && Number.isInteger(value.foreground_pid) && value.foreground_pid !== foregroundPid) return false;+ // A pointer press must land on the pressed button (screen pixels); a key press must be one that presses a button.+ if (value.kind === "pointer" && rect) {+ if (!Number.isFinite(value.x) || !Number.isFinite(value.y)) return false;+ return value.x >= rect.x - 4 && value.x <= rect.x + rect.width + 4 && value.y >= rect.y - 4 && value.y <= rect.y + rect.height + 4;+ }+ if (value.kind === "key") return keys.includes(value.key);+ return !rect;}async function syntheticInput(withinMs = 5000) {diff --git a/packages/computer-use/native/Sources/ComputerUse/NativeWorker.swift b/packages/computer-use/native/Sources/ComputerUse/NativeWorker.swiftindex 5adbd810..132139d8 100644--- a/packages/computer-use/native/Sources/ComputerUse/NativeWorker.swift+++ b/packages/computer-use/native/Sources/ComputerUse/NativeWorker.swift@@ -600,12 +600,35 @@ final class NativeWorker {}if [.leftMouseDown, .rightMouseDown, .otherMouseDown, .keyDown].contains(event.type) {let stateID = cg?.getIntegerValueField(.eventSourceStateID) ?? -1- lastInput = ["synthetic": WorkerInputSource.isSynthetic(sourcePid: sourcePid, stateID: stateID, sourceIsWindowServer: Self.isWindowServer(pid_t(truncatingIfNeeded: sourcePid))),- "at": Int(Date().timeIntervalSince1970 * 1000), "kind": event.type == .keyDown ? "key" : "pointer"]+ let synthetic = WorkerInputSource.isSynthetic(sourcePid: sourcePid, stateID: stateID, sourceIsWindowServer: Self.isWindowServer(pid_t(truncatingIfNeeded: sourcePid)))+ var entry: [String: Any] = ["synthetic": synthetic, "at": Int(Date().timeIntervalSince1970 * 1000), "kind": event.type == .keyDown ? "key" : "pointer"]+ // The app in front at the press, where the pointer was (global points, top-left origin) and the key as a terminal names it.+ entry["foreground_pid"] = NSWorkspace.shared.frontmostApplication.map { Int($0.processIdentifier) } ?? NSNull()+ if let at = cg?.location, event.type != .keyDown { entry["x"] = Double(at.x); entry["y"] = Double(at.y) }+ entry["key"] = event.type == .keyDown ? (Self.keyName(event) ?? NSNull()) : NSNull()+ lastInput = entry}personInput(force: false)}+ /** One lower-case letter or digit for printable keys, else Enter / Escape / Space / Tab / Backspace / arrows; nil otherwise. */+ static func keyName(_ event: NSEvent) -> String? {+ switch event.keyCode {+ case 36, 76: return "Enter"+ case 53: return "Escape"+ case 49: return "Space"+ case 48: return "Tab"+ case 51: return "Backspace"+ case 123: return "Left"+ case 124: return "Right"+ case 125: return "Down"+ case 126: return "Up"+ default:+ guard let chars = event.charactersIgnoringModifiers?.lowercased(), chars.count == 1, let c = chars.first, c.isLetter || c.isNumber else { return nil }+ return String(c)+ }+ }+private static func isWindowServer(_ pid: pid_t) -> Bool {guard pid > 0 else { return false }var name = [CChar](repeating: 0, count: 256)diff --git a/packages/types/src/desktop-ipc.ts b/packages/types/src/desktop-ipc.tsindex 89fe8488..a15a3584 100644--- a/packages/types/src/desktop-ipc.ts+++ b/packages/types/src/desktop-ipc.ts@@ -936,7 +936,9 @@ export type ComputerUseRequest =chat: string | null;};-export type ComputerUseAnswer = { request_id: string; decision: "chat" | "once" | "deny" | "allow"; window_id?: string };+/** The pressed button's box (CSS pixels of the main window): the person's own press must land in it. */+export type ComputerUseButtonRect = { x: number; y: number; width: number; height: number };+export type ComputerUseAnswer = { request_id: string; decision: "chat" | "once" | "deny" | "allow"; window_id?: string; rect?: ComputerUseButtonRect };export type ComputerUseControlAction = "stop" | "takeover" | "continue" | "hide" | "show";export type ComputerUseStatus = {@@ -1037,7 +1039,7 @@ export type DesktopCommandMap = {getComputerUseMcpEnvironment: { args: []; result: Record<string, string> };getComputerUseState: { args: []; result: ComputerUseStatus };computerUseAnswer: { args: [answer: ComputerUseAnswer]; result: void };- computerUseControl: { args: [value: { session_id: string; action: ComputerUseControlAction }]; result: void };+ computerUseControl: { args: [value: { session_id: string; action: ComputerUseControlAction; rect?: ComputerUseButtonRect }]; result: void };getOmniRushUiMcpEnvironment: { args: []; result: Record<string, string> };// Computer use