Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: answers need a press on the button

ogsamratdc898cf00d3f
diff --git a/apps/app/src/react-app/domains/session/surface/computer-use/live-panel.tsx b/apps/app/src/react-app/domains/session/surface/computer-use/live-panel.tsx
index a2635992..27a13a1d 100644
--- a/apps/app/src/react-app/domains/session/surface/computer-use/live-panel.tsx
+++ b/apps/app/src/react-app/domains/session/surface/computer-use/live-panel.tsx
@@ -7,7 +7,7 @@ import { stripRemoteMethodErrorPrefix } from "@/react-app/domains/settings/state
import { formatMinutesLeft } from "./describe";
import { ComputerUseAppIdentity, ComputerUseFrame } from "./request-card";
-import { controlComputerUse } from "./status-store";
+import { buttonRect, controlComputerUse } from "./status-store";
/** A running (or paused) computer use session: what it sees, what it did, and Stop. */
export function ComputerUseLivePanel(props: { session: ComputerUseSessionState }) {
@@ -17,10 +17,10 @@ export function ComputerUseLivePanel(props: { session: ComputerUseSessionState }
const [error, setError] = useState<string | null>(null);
const paused = session.phase === "paused";
- const control = (action: ComputerUseControlAction) => {
+ const control = (action: ComputerUseControlAction, event?: { currentTarget: EventTarget | null }) => {
setBusy(true);
setError(null);
- controlComputerUse(session.id, action)
+ controlComputerUse(session.id, action, buttonRect(event?.currentTarget ?? null))
.catch((reason: unknown) => setError(reason instanceof Error ? stripRemoteMethodErrorPrefix(reason.message) : "That did not work. Try again."))
.finally(() => setBusy(false));
};
@@ -28,18 +28,18 @@ export function ComputerUseLivePanel(props: { session: ComputerUseSessionState }
const buttons = (
<div className="flex shrink-0 flex-wrap items-center gap-2">
{paused && session.can_continue ? (
- <Button type="button" variant="outline" size="xs" disabled={busy} onClick={() => control("continue")} data-computer-use-control="continue">
+ <Button type="button" variant="outline" size="xs" disabled={busy} onClick={(event) => control("continue", event)} data-computer-use-control="continue">
<Play data-icon="inline-start" />
Continue
</Button>
) : null}
{!paused && session.mode === "control" ? (
- <Button type="button" variant="outline" size="xs" disabled={busy} onClick={() => control("takeover")} data-computer-use-control="takeover">
+ <Button type="button" variant="outline" size="xs" disabled={busy} onClick={(event) => control("takeover", event)} data-computer-use-control="takeover">
<Hand data-icon="inline-start" />
Take over
</Button>
) : null}
- <Button type="button" variant="destructive" size="xs" disabled={busy} onClick={() => control("stop")} data-computer-use-control="stop">
+ <Button type="button" variant="destructive" size="xs" disabled={busy} onClick={(event) => control("stop", event)} data-computer-use-control="stop">
<Square data-icon="inline-start" />
Stop
</Button>
diff --git a/apps/app/src/react-app/domains/session/surface/computer-use/request-card.tsx b/apps/app/src/react-app/domains/session/surface/computer-use/request-card.tsx
index 02746631..9db5a30b 100644
--- a/apps/app/src/react-app/domains/session/surface/computer-use/request-card.tsx
+++ b/apps/app/src/react-app/domains/session/surface/computer-use/request-card.tsx
@@ -6,7 +6,7 @@ import { Button } from "@/components/ui/button";
import { stripRemoteMethodErrorPrefix } from "@/react-app/domains/settings/state/electron-updater-state";
import { describeComputerUseAction, scalePoints, typedText, type ComputerUsePoint } from "./describe";
-import { answerComputerUse } from "./status-store";
+import { answerComputerUse, buttonRect } from "./status-store";
const APPROVE_WITH_OWN_INPUT = "Use your own mouse or keyboard to approve.";
@@ -88,10 +88,11 @@ export function ComputerUseRequestCard(props: { request: ComputerUseRequest }) {
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
- const answer = (decision: ComputerUseAnswer["decision"]) => {
+ const answer = (decision: ComputerUseAnswer["decision"], event?: { currentTarget: EventTarget | null }) => {
setBusy(true);
setError(null);
- answerComputerUse({ request_id: request.request_id, decision, ...(request.kind === "open" ? { window_id: windowId } : {}) })
+ const rect = buttonRect(event?.currentTarget ?? null);
+ answerComputerUse({ request_id: request.request_id, decision, ...(request.kind === "open" ? { window_id: windowId } : {}), ...(rect ? { rect } : {}) })
.catch((reason: unknown) => {
const message = reason instanceof Error ? stripRemoteMethodErrorPrefix(reason.message) : "";
setError(message || APPROVE_WITH_OWN_INPUT);
@@ -140,18 +141,18 @@ export function ComputerUseRequestCard(props: { request: ComputerUseRequest }) {
</p>
{errorLine}
<div className="flex flex-wrap items-center gap-2">
- <Button type="button" size="sm" disabled={busy} onClick={() => answer("chat")} data-computer-use-answer="chat" className="h-auto py-1.5">
+ <Button type="button" size="sm" disabled={busy} onClick={(event) => answer("chat", event)} data-computer-use-answer="chat" className="h-auto py-1.5">
<Check data-icon="inline-start" />
<span className="flex flex-col items-start leading-4">
<span>Allow for this chat</span>
<span className="text-[10px] font-normal opacity-80">Same app, up to {Math.round(request.chat_seconds / 60)} minutes</span>
</span>
</Button>
- <Button type="button" variant="outline" size="sm" disabled={busy} onClick={() => answer("once")} data-computer-use-answer="once">
+ <Button type="button" variant="outline" size="sm" disabled={busy} onClick={(event) => answer("once", event)} data-computer-use-answer="once">
<Clock3 data-icon="inline-start" />
Allow once
</Button>
- <Button type="button" variant="destructive" size="sm" disabled={busy} onClick={() => answer("deny")} data-computer-use-answer="deny">
+ <Button type="button" variant="destructive" size="sm" disabled={busy} onClick={(event) => answer("deny", event)} data-computer-use-answer="deny">
<XCircle data-icon="inline-start" />
Deny
</Button>
@@ -185,11 +186,11 @@ export function ComputerUseRequestCard(props: { request: ComputerUseRequest }) {
<ComputerUseFrame frame={request.frame} points={request.points} alt={`${request.app.app_name} window`} />
{errorLine}
<div className="flex flex-wrap items-center gap-2">
- <Button type="button" size="sm" disabled={busy} onClick={() => answer("allow")} data-computer-use-answer="allow">
+ <Button type="button" size="sm" disabled={busy} onClick={(event) => answer("allow", event)} data-computer-use-answer="allow">
<Check data-icon="inline-start" />
Allow
</Button>
- <Button type="button" variant="destructive" size="sm" disabled={busy} onClick={() => answer("deny")} data-computer-use-answer="deny">
+ <Button type="button" variant="destructive" size="sm" disabled={busy} onClick={(event) => answer("deny", event)} data-computer-use-answer="deny">
<XCircle data-icon="inline-start" />
Deny
</Button>
diff --git a/apps/app/src/react-app/domains/session/surface/computer-use/status-store.ts b/apps/app/src/react-app/domains/session/surface/computer-use/status-store.ts
index ec4fdb87..1667d618 100644
--- a/apps/app/src/react-app/domains/session/surface/computer-use/status-store.ts
+++ b/apps/app/src/react-app/domains/session/surface/computer-use/status-store.ts
@@ -5,7 +5,14 @@
import { useEffect, useSyncExternalStore } from "react";
import { create } from "zustand";
-import type { ComputerUseAnswer, ComputerUseControlAction, ComputerUseStatus } from "@omnirush/types/desktop-ipc";
+import type { ComputerUseAnswer, ComputerUseButtonRect, ComputerUseControlAction, ComputerUseStatus } from "@omnirush/types/desktop-ipc";
+
+/** The pressed button's box, for main's check that the person pressed it themselves. */
+export function buttonRect(target: EventTarget | null): ComputerUseButtonRect | undefined {
+ if (!(target instanceof Element)) return undefined;
+ const r = target.getBoundingClientRect();
+ return { x: r.x, y: r.y, width: r.width, height: r.height };
+}
let current: ComputerUseStatus | null = null;
const listeners = new Set<() => void>();
@@ -81,8 +88,8 @@ export async function answerComputerUse(answer: ComputerUseAnswer) {
}
/** Stop, take over or continue a session. */
-export async function controlComputerUse(session_id: string, action: ComputerUseControlAction) {
+export async function controlComputerUse(session_id: string, action: ComputerUseControlAction, rect?: ComputerUseButtonRect) {
const invoke = window.__OMNIRUSH_ELECTRON__?.invokeDesktop;
if (!invoke) throw new Error("Computer use needs the desktop app.");
- await invoke("computerUseControl", { session_id, action });
+ await invoke("computerUseControl", { session_id, action, ...(rect ? { rect } : {}) });
}
diff --git a/apps/desktop/electron/computer-use.mjs b/apps/desktop/electron/computer-use.mjs
index 31ec1902..d3cb6f39 100644
--- a/apps/desktop/electron/computer-use.mjs
+++ b/apps/desktop/electron/computer-use.mjs
@@ -18,7 +18,7 @@ import { createRequire } from "node:module";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
-import { app } from "electron";
+import { app, screen } from "electron";
import { testAnswersAllowed } from "./computer-use-answers.mjs";
import { createComputerUseHost } from "./computer-use-host.mjs";
import { createComputerPreview } from "./computer-use-preview.mjs";
@@ -38,10 +38,25 @@ const SYNTHETIC_MESSAGE = "Use your own mouse or keyboard to approve.";
// remote-debugging clicks, in builds run from source only (the e2e rig).
// Packaged apps ignore it (computer-use-answers.mjs).
const testAnswers = testAnswersAllowed(process.env, app.isPackaged, (message) => console.warn(`[computer-use] ${message}`));
-async function personAnswered() {
+// The pressed button in screen pixels (the worker's press coordinates): content bounds + CSS box × zoom.
+function screenRectOf(rect) {
+ const window = deps?.getWindow?.();
+ if (!window || window.isDestroyed() || !rect || ![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite)) return null;
+ const content = window.getContentBounds(), zoom = window.webContents.getZoomFactor();
+ const dip = { x: Math.round(content.x + rect.x * zoom), y: Math.round(content.y + rect.y * zoom), width: Math.round(rect.width * zoom), height: Math.round(rect.height * zoom) };
+ if (process.platform === "win32") return screen.dipToScreenRect(window, dip);
+ if (process.platform === "linux") {
+ const factor = screen.getDisplayMatching(dip).scaleFactor || 1;
+ return { x: dip.x * factor, y: dip.y * factor, width: dip.width * factor, height: dip.height * factor };
+ }
+ return dip; // macOS: points, as the helper reports presses
+}
+
+// A physical press within 1.5 s, in this app's window, on the pressed button (or Enter/Space on it).
+async function personAnswered(rect) {
if (!host) return false;
if (await host.syntheticInput()) return false;
- return testAnswers || host.physicalInput(5000, { foregroundPid: process.pid });
+ return testAnswers || host.physicalInput(1500, { foregroundPid: process.pid, rect: screenRectOf(rect) });
}
let deps = null;
@@ -259,14 +274,14 @@ async function computerUseAnswer(answer) {
const entry = pending.get(answer?.request_id);
if (!entry) throw new Error("This request has ended.");
const allowing = answer.decision !== "deny";
- if (allowing && !(await personAnswered())) throw new Error(SYNTHETIC_MESSAGE);
+ if (allowing && !(await personAnswered(answer.rect))) throw new Error(SYNTHETIC_MESSAGE);
const decision = entry.request.kind === "open" ? (["chat", "once", "deny"].includes(answer.decision) ? answer.decision : "deny") : (answer.decision === "allow" ? "allow" : "deny");
entry.resolve({ decision, ...(typeof answer.window_id === "string" ? { window_id: answer.window_id } : {}) });
}
async function computerUseControl(value) {
if (!host) throw new Error("Computer use is not running.");
- if (value?.action === "continue" && !(await personAnswered())) throw new Error(SYNTHETIC_MESSAGE);
+ if (value?.action === "continue" && !(await personAnswered(value.rect))) throw new Error(SYNTHETIC_MESSAGE);
await host.control({ session_id: value?.session_id, action: value?.action });
pushStatus();
}
diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.ts
index 1b2f5391..600a77d8 100644
--- a/apps/server/src/computer-use/portable-host.test.ts
+++ b/apps/server/src/computer-use/portable-host.test.ts
@@ -24,7 +24,7 @@ function fakeNative() {
const listeners = new Set<(v: Record<string, unknown>) => void>();
const calls: Array<{ method: string; params: Record<string, unknown> }> = [];
let generation = 0;
- const state: { occluded: boolean; synthetic: boolean; retitle: ((title: string) => string) | null; texts: Array<{ x: number; y: number; width: number; height: number; text: string }> | null } = { occluded: false, synthetic: false, retitle: null, texts: null };
+ const state: { occluded: boolean; synthetic: boolean; retitle: ((title: string) => string) | null; texts: Array<{ x: number; y: number; width: number; height: number; text: string }> | null; pressAge?: number; pressKind?: string; pressKey?: string; pressAt?: { x: number; y: number }; pressPid?: number } = { occluded: false, synthetic: false, retitle: null, texts: null };
const frame = (w: Window): Frame => {
const width = Math.min(w.bounds.width, 640), height = Math.round((width * w.bounds.height) / w.bounds.width);
const data = new Uint8Array(width * height * 4).fill(shade);
@@ -52,7 +52,7 @@ function fakeNative() {
if (method === "state") return { generation, available: true };
if (method === "start") return { generation };
if (method === "hit") return { inside: (params.points as unknown[]).map(() => !state.occluded) };
- if (method === "input-source") return { synthetic: state.synthetic, at: Date.now(), kind: "pointer" };
+ if (method === "input-source") return { synthetic: state.synthetic, at: Date.now() - (state.pressAge ?? 0), kind: state.pressKind ?? "pointer", key: state.pressKey ?? null, x: state.pressAt?.x ?? null, y: state.pressAt?.y ?? null, foreground_pid: state.pressPid ?? null };
if (method === "act") {
if (params.generation !== generation) throw Object.assign(new Error("Your input paused computer use."), { code: "paused_by_person" });
if (state.retitle) windows[0]!.title = state.retitle(windows[0]!.title);
@@ -369,6 +369,27 @@ describe("portable host: titles that follow the agent's own typing, and a worker
});
});
+describe("portable host: the person's own press answers a question", () => {
+ test("within 1.5 s, in the app's window, on the pressed button or with Enter/Space", async () => {
+ const r = rig("normal");
+ const host = await r.hostPromise;
+ const rect = { x: 100, y: 200, width: 80, height: 30 };
+ r.native.state.pressAt = { x: 120, y: 210 }; r.native.state.pressPid = 4242;
+ expect(await host.physicalInput(1500, { foregroundPid: 4242, rect })).toBe(true);
+ r.native.state.pressAt = { x: 400, y: 210 };
+ expect(await host.physicalInput(1500, { foregroundPid: 4242, rect })).toBe(false);
+ r.native.state.pressAt = { x: 120, y: 210 };
+ expect(await host.physicalInput(1500, { foregroundPid: 999, rect })).toBe(false);
+ r.native.state.pressAge = 2000;
+ expect(await host.physicalInput(1500, { foregroundPid: 4242, rect })).toBe(false);
+ r.native.state.pressAge = 0; r.native.state.pressKind = "key"; r.native.state.pressKey = "Enter";
+ expect(await host.physicalInput(1500, { foregroundPid: 4242, rect })).toBe(true);
+ r.native.state.pressKey = "y";
+ expect(await host.physicalInput(1500, { foregroundPid: 4242, rect })).toBe(false);
+ await host.close();
+ });
+});
+
describe("portable host: the control under the point from the worker (Windows MSAA)", () => {
test("a named control decides (risky target, protected field); no name falls back to target_unknown", async () => {
const r = rig("normal");
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256
index 940d1332..f96eef5b 100644
--- a/apps/server/src/computer-use/shared/PARITY.sha256
+++ b/apps/server/src/computer-use/shared/PARITY.sha256
@@ -8,6 +8,6 @@ a96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs
5eac31868aedd3e1d1e71b445f9a55d01316da7397f27f6db11472453bee4588 image.d.mts
e47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjs
eabdf78c56733577d05aad490d9d7d5ece6c22e9b3ea9e6e9803d1a4a546fdd4 media.d.mts
-8543b410d31a33e700f4f6dbde3547ad0caa92421af387fe3e772a7444edfd60 native-worker.mjs
-7a55a915d4d725a4e992e7ef6191ba0ad1c851cdb5ee712ef79e0f7981a6a1c4 portable-host.mjs
-2ed7be072df62d50e1b569e75b695d18ae0d0de6dabe5375bc494b859cf78710 portable-host.d.mts
+a6a6596c75df17c0aac6d7c740ae9661cae2b8899d12e20b5c2f37987cf9d4be native-worker.mjs
+110b5f3aadaaec96b523fde230fa513d0fd75f2f7634b7336ac5b4ba975f18f4 portable-host.mjs
+641be80cebb04d353031392d8f53f45ead1e1beaf83f1e67d24945f0eda330bf portable-host.d.mts
diff --git a/apps/server/src/computer-use/shared/native-worker.mjs b/apps/server/src/computer-use/shared/native-worker.mjs
index 31db5551..bec6109e 100644
--- a/apps/server/src/computer-use/shared/native-worker.mjs
+++ b/apps/server/src/computer-use/shared/native-worker.mjs
@@ -32,12 +32,12 @@ const presses = [];
function notePress(entry) {
Object.assign(lastInput, { synthetic: entry.synthetic, at: entry.at, kind: entry.kind });
if (entry.synthetic) return;
- presses.push({ at: entry.at, kind: entry.kind, key: entry.key ?? null, foreground_pid: entry.foreground_pid ?? null });
+ presses.push({ at: entry.at, kind: entry.kind, key: entry.key ?? null, foreground_pid: entry.foreground_pid ?? null, x: entry.x ?? null, y: entry.y ?? null });
if (presses.length > 16) presses.shift();
}
function inputSource() {
const physical = presses.at(-1) ?? null;
- return { ...lastInput, key: physical?.key ?? null, foreground_pid: physical?.foreground_pid ?? null, physical_at: physical?.at ?? 0, presses: presses.slice() };
+ return { ...lastInput, key: physical?.key ?? null, foreground_pid: physical?.foreground_pid ?? null, x: physical?.x ?? null, y: physical?.y ?? null, physical_at: physical?.at ?? 0, presses: presses.slice() };
}
const changed = () => { generation += 1; post({ event: "person-input", generation }); };
const fail = (code, message) => { throw Object.assign(new Error(message), { code }); };
@@ -169,7 +169,8 @@ function windows() {
if (down) {
const fg = [0];
try { pidOf(foreground(), fg); } catch { /* unknown */ }
- notePress({ synthetic: (value.flags & injectedFlag) !== 0, at: Date.now(), kind: type === hookKey ? "key" : "pointer", key: type === hookKey ? keyFromVirtualKey(Number(value.vkCode)) : null, foreground_pid: fg[0] || null });
+ const at = type === hookMouse ? { x: value.pt.x, y: value.pt.y } : {};
+ notePress({ synthetic: (value.flags & injectedFlag) !== 0, at: Date.now(), kind: type === hookKey ? "key" : "pointer", key: type === hookKey ? keyFromVirtualKey(Number(value.vkCode)) : null, foreground_pid: fg[0] || null, ...at });
}
if (interrupting) { changed(); release(); }
}
@@ -570,12 +571,13 @@ function x11() {
if (ours) continue;
if (cookie.evtype === 13 || cookie.evtype === 15) {
const synthetic = testDevices.has(raw.sourceid);
- let key = null, fg = null;
+ let key = null, fg = null, where = {};
if (!synthetic) {
+ try { if (cookie.evtype === 15) { const r = [0], c = [0], rx = [0], ry = [0], wx = [0], wy = [0], mk = [0]; if (queryPointer(display, root, r, c, rx, ry, wx, wy, mk)) where = { x: rx[0], y: ry[0] }; } } catch { /* unknown */ }
try { if (cookie.evtype === 13) key = keyFromKeysymName(keysymName(keycodeKeysym(display, raw.detail, 0, 0))); } catch { /* unknown */ }
try { fg = Number(prop(activeWindow(), "_NET_WM_PID")[0] ?? 0) || null; } catch { /* unknown */ }
}
- notePress({ synthetic, at: Date.now(), kind: cookie.evtype === 13 ? "key" : "pointer", key, foreground_pid: fg });
+ notePress({ synthetic, at: Date.now(), kind: cookie.evtype === 13 ? "key" : "pointer", key, foreground_pid: fg, ...where });
}
if (interrupting) changed();
}
diff --git a/apps/server/src/computer-use/shared/portable-host.d.mts b/apps/server/src/computer-use/shared/portable-host.d.mts
index bfb78b6d..a569c864 100644
--- a/apps/server/src/computer-use/shared/portable-host.d.mts
+++ b/apps/server/src/computer-use/shared/portable-host.d.mts
@@ -80,7 +80,8 @@ export type PortableHost = {
stopAll(by?: SessionEvent["by"], event?: SessionEvent["event"]): Promise<void>;
syntheticInput(withinMs?: number): Promise<boolean>;
/** A physical key or button press within `withinMs` (not injected, not a remote-debugging click). */
- physicalInput(withinMs?: number, options?: { foregroundPid?: number | null }): Promise<boolean>;
+ /** Default 1.5 s; a pointer press must be inside `rect` (screen pixels) when given; a key press must be one of `keys` (default Enter, Space). */
+ physicalInput(withinMs?: number, options?: { foregroundPid?: number | null; rect?: { x: number; y: number; width: number; height: number } | null; keys?: string[] }): Promise<boolean>;
watchInput(): Promise<void>;
state(): HostSessionState[];
close(): Promise<void>;
diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjs
index 674ddd76..7d8f8fd8 100644
--- a/apps/server/src/computer-use/shared/portable-host.mjs
+++ b/apps/server/src/computer-use/shared/portable-host.mjs
@@ -656,11 +656,18 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
* Whether a physical key or button press (not injected by a program, not a remote-debugging
* click that makes no OS input) came within `withinMs`: the person's own answer.
*/
- async function physicalInput(withinMs = 5000, { foregroundPid = null } = {}) {
+ async function physicalInput(withinMs = 1500, { foregroundPid = null, rect = null, keys = ["Enter", "Space"] } = {}) {
const value = await native.call("input-source").catch(() => null);
if (!value || !value.at || value.synthetic || Date.now() - value.at >= withinMs) return false;
// In the answering app's window (when the worker knows which app was in front).
- return !(foregroundPid && Number.isInteger(value.foreground_pid) && value.foreground_pid !== foregroundPid);
+ if (foregroundPid && Number.isInteger(value.foreground_pid) && value.foreground_pid !== foregroundPid) return false;
+ // A pointer press must land on the pressed button (screen pixels); a key press must be one that presses a button.
+ if (value.kind === "pointer" && rect) {
+ if (!Number.isFinite(value.x) || !Number.isFinite(value.y)) return false;
+ return value.x >= rect.x - 4 && value.x <= rect.x + rect.width + 4 && value.y >= rect.y - 4 && value.y <= rect.y + rect.height + 4;
+ }
+ if (value.kind === "key") return keys.includes(value.key);
+ return !rect;
}
async function syntheticInput(withinMs = 5000) {
diff --git a/packages/computer-use/native/Sources/ComputerUse/NativeWorker.swift b/packages/computer-use/native/Sources/ComputerUse/NativeWorker.swift
index 5adbd810..132139d8 100644
--- a/packages/computer-use/native/Sources/ComputerUse/NativeWorker.swift
+++ b/packages/computer-use/native/Sources/ComputerUse/NativeWorker.swift
@@ -600,12 +600,35 @@ final class NativeWorker {
}
if [.leftMouseDown, .rightMouseDown, .otherMouseDown, .keyDown].contains(event.type) {
let stateID = cg?.getIntegerValueField(.eventSourceStateID) ?? -1
- lastInput = ["synthetic": WorkerInputSource.isSynthetic(sourcePid: sourcePid, stateID: stateID, sourceIsWindowServer: Self.isWindowServer(pid_t(truncatingIfNeeded: sourcePid))),
- "at": Int(Date().timeIntervalSince1970 * 1000), "kind": event.type == .keyDown ? "key" : "pointer"]
+ let synthetic = WorkerInputSource.isSynthetic(sourcePid: sourcePid, stateID: stateID, sourceIsWindowServer: Self.isWindowServer(pid_t(truncatingIfNeeded: sourcePid)))
+ var entry: [String: Any] = ["synthetic": synthetic, "at": Int(Date().timeIntervalSince1970 * 1000), "kind": event.type == .keyDown ? "key" : "pointer"]
+ // The app in front at the press, where the pointer was (global points, top-left origin) and the key as a terminal names it.
+ entry["foreground_pid"] = NSWorkspace.shared.frontmostApplication.map { Int($0.processIdentifier) } ?? NSNull()
+ if let at = cg?.location, event.type != .keyDown { entry["x"] = Double(at.x); entry["y"] = Double(at.y) }
+ entry["key"] = event.type == .keyDown ? (Self.keyName(event) ?? NSNull()) : NSNull()
+ lastInput = entry
}
personInput(force: false)
}
+ /** One lower-case letter or digit for printable keys, else Enter / Escape / Space / Tab / Backspace / arrows; nil otherwise. */
+ static func keyName(_ event: NSEvent) -> String? {
+ switch event.keyCode {
+ case 36, 76: return "Enter"
+ case 53: return "Escape"
+ case 49: return "Space"
+ case 48: return "Tab"
+ case 51: return "Backspace"
+ case 123: return "Left"
+ case 124: return "Right"
+ case 125: return "Down"
+ case 126: return "Up"
+ default:
+ guard let chars = event.charactersIgnoringModifiers?.lowercased(), chars.count == 1, let c = chars.first, c.isLetter || c.isNumber else { return nil }
+ return String(c)
+ }
+ }
+
private static func isWindowServer(_ pid: pid_t) -> Bool {
guard pid > 0 else { return false }
var name = [CChar](repeating: 0, count: 256)
diff --git a/packages/types/src/desktop-ipc.ts b/packages/types/src/desktop-ipc.ts
index 89fe8488..a15a3584 100644
--- a/packages/types/src/desktop-ipc.ts
+++ b/packages/types/src/desktop-ipc.ts
@@ -936,7 +936,9 @@ export type ComputerUseRequest =
chat: string | null;
};
-export type ComputerUseAnswer = { request_id: string; decision: "chat" | "once" | "deny" | "allow"; window_id?: string };
+/** The pressed button's box (CSS pixels of the main window): the person's own press must land in it. */
+export type ComputerUseButtonRect = { x: number; y: number; width: number; height: number };
+export type ComputerUseAnswer = { request_id: string; decision: "chat" | "once" | "deny" | "allow"; window_id?: string; rect?: ComputerUseButtonRect };
export type ComputerUseControlAction = "stop" | "takeover" | "continue" | "hide" | "show";
export type ComputerUseStatus = {
@@ -1037,7 +1039,7 @@ export type DesktopCommandMap = {
getComputerUseMcpEnvironment: { args: []; result: Record<string, string> };
getComputerUseState: { args: []; result: ComputerUseStatus };
computerUseAnswer: { args: [answer: ComputerUseAnswer]; result: void };
- computerUseControl: { args: [value: { session_id: string; action: ComputerUseControlAction }]; result: void };
+ computerUseControl: { args: [value: { session_id: string; action: ComputerUseControlAction; rect?: ComputerUseButtonRect }]; result: void };
getOmniRushUiMcpEnvironment: { args: []; result: Record<string, string> };
// Computer use