Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: ask before clicks on unnamed controls

ogsamrat2c6d2734c3e6
diff --git a/apps/server/src/computer-use/contract-policy.test.ts b/apps/server/src/computer-use/contract-policy.test.ts
index 6d7f8182..6a98fce0 100644
--- a/apps/server/src/computer-use/contract-policy.test.ts
+++ b/apps/server/src/computer-use/contract-policy.test.ts
@@ -250,10 +250,12 @@ describe("policy: approvals per permission mode", () => {
expect(documentOf("● main.ts - Code", "Code")).toBe("main.ts");
});
- test("Normal, no readable text at the target: a click or Enter right after typing asks", () => {
- expect(actApproval({ ...plain, target: {}, typed_before: true }).reasons).toEqual([{ code: "submit_after_typing" }]);
- expect(actApproval({ ...plain, action: { type: "key", key: "Enter" }, target: {}, typed_before: true }).ask).toBe(true);
- expect(actApproval({ ...plain, target: {}, typed_before: false }).ask).toBe(false);
+ test("Normal, no readable text at the target: a click asks (target unknown); Enter right after typing asks", () => {
+ expect(actApproval({ ...plain, target: {} }).reasons).toEqual([{ code: "target_unknown" }]);
+ expect(actApproval({ ...plain, action: { type: "right_click", x: 1, y: 1 }, target: {} }).ask).toBe(true);
+ expect(actApproval({ ...plain, action: { type: "key", key: "Enter" }, target: {}, typed_before: true }).reasons).toEqual([{ code: "submit_after_typing" }]);
+ expect(actApproval({ ...plain, action: { type: "key", key: "enter" }, target: {}, typed_before: false }).ask).toBe(false);
+ expect(actApproval({ ...plain, action: { type: "move", x: 1, y: 1 }, target: {} }).ask).toBe(false);
// With text the host read there, the target rules decide instead.
expect(actApproval({ ...plain, target: { text: "Bold" }, typed_before: true }).ask).toBe(false);
expect(actApproval({ ...plain, mode: "auto", target: {}, typed_before: true }).ask).toBe(false);
diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.ts
index fe3d77ef..1b2f5391 100644
--- a/apps/server/src/computer-use/portable-host.test.ts
+++ b/apps/server/src/computer-use/portable-host.test.ts
@@ -200,6 +200,7 @@ describe("portable host: observe and act", () => {
test("Normal: plain acts run, risky ones ask", async () => {
const r = rig("normal");
+ r.native.state.texts = [{ x: 0, y: 0, width: 100, height: 100, text: "Notes" }];
const { host, c, session, obs } = await opened(r);
const plain = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Put the cursor at the end", action: { type: "click", x: 5, y: 5 } });
expect(plain.state).toMatchObject({ ok: true, approval: { asked: false } });
@@ -259,14 +260,15 @@ describe("portable host: observe and act", () => {
const secure = await call(c, "computer_act", { session_id: session, observation_id: (sign.state.observation as { id: string }).id, request_id: "b", intent: "Fill in the field", action: { type: "type", text: "abc" } });
expect(secure.state).toMatchObject({ ok: true, approval: { asked: false } });
await host.close();
- // A host that reads no text (X11, Windows): type, then a click asks.
+ // A host that reads no text (X11): every click asks (the app doesn't say what it is); typing does not.
const x = rig("normal");
const o = await opened(x);
const typed = await call(o.c, "computer_act", { session_id: o.session, observation_id: o.obs.state.observation?.id, request_id: "t", intent: "Type the name", action: { type: "type", text: "Ada" } });
expect(typed.state).toMatchObject({ ok: true, approval: { asked: false } });
const click = await call(o.c, "computer_act", { session_id: o.session, observation_id: (typed.state.observation as { id: string }).id, request_id: "k", intent: "Go to the next field", action: { type: "click", x: 400, y: 300 } });
- expect(click.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["submit_after_typing"] } });
- const again = await call(o.c, "computer_act", { session_id: o.session, observation_id: (click.state.observation as { id: string }).id, request_id: "k2", intent: "Click again", action: { type: "click", x: 400, y: 300 } });
+ expect(click.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["target_unknown"] } });
+ // The same click on the same image within 30 s (a retry) is not asked again.
+ const again = await call(o.c, "computer_act", { session_id: o.session, observation_id: (click.state.observation as { id: string }).id, request_id: "k2", intent: "Go to the next field", action: { type: "click", x: 400, y: 300 } });
expect(again.state).toMatchObject({ ok: true, approval: { asked: false } });
expect(await o.host.physicalInput()).toBe(true);
x.native.state.synthetic = true;
@@ -284,7 +286,7 @@ describe("portable host: observe and act", () => {
await host.control({ session_id: session, action: "continue" });
const next = await call(c, "computer_observe", { session_id: session });
const act = await call(c, "computer_act", { session_id: session, observation_id: next.state.observation?.id, request_id: "b", intent: "Click in the text", action: { type: "click", x: 5, y: 5 } });
- expect(act.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["after_takeover"] } });
+ expect(act.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["target_unknown", "after_takeover"] } });
expect(r.events.map((e) => `${e.event}:${e.by}`)).toEqual(["opened:agent", "approved:person", "paused:person", "continued:person"]);
await host.control({ session_id: session, action: "stop" });
expect(r.events.at(-1)).toMatchObject({ event: "stopped", by: "person", app_id: "/usr/bin/gedit", app_name: "gedit", bundle_or_exe: "/usr/bin/gedit", mode: "control", approval: "once", os: "linux", screen: { id: "0", width: 3840, height: 2160 } });
@@ -367,6 +369,25 @@ describe("portable host: titles that follow the agent's own typing, and a worker
});
});
+describe("portable host: the control under the point from the worker (Windows MSAA)", () => {
+ test("a named control decides (risky target, protected field); no name falls back to target_unknown", async () => {
+ const r = rig("normal");
+ let next: { name: string; role: number | null; protected: boolean } | null = { name: "Pay now", role: 43, protected: false };
+ const call0 = r.native.call.bind(r.native);
+ r.native.call = async (method: string, params: Record<string, unknown> = {}) => (method === "target" ? { targets: [next] } : call0(method, params));
+ const { host, c, session, obs } = await opened(r);
+ const pay = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "p", intent: "Press the button", action: { type: "click", x: 10, y: 10 } });
+ expect(pay.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["risky_target"] } });
+ next = { name: "Bold", role: 43, protected: false };
+ const bold = await call(c, "computer_act", { session_id: session, observation_id: (pay.state.observation as { id: string }).id, request_id: "b", intent: "Make it bold", action: { type: "click", x: 20, y: 10 } });
+ expect(bold.state).toMatchObject({ ok: true, approval: { asked: false } });
+ next = { name: "", role: 42, protected: false };
+ const blank = await call(c, "computer_act", { session_id: session, observation_id: (bold.state.observation as { id: string }).id, request_id: "n", intent: "Click there", action: { type: "click", x: 30, y: 10 } });
+ expect(blank.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["target_unknown"] } });
+ await host.close();
+ });
+});
+
describe("portable host on Windows: the agent's ancestors come from the worker", () => {
test("an IDE the agent runs in is denied as an ancestor when the app passed none", async () => {
const native = fakeNative();
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256
index 47262c32..940d1332 100644
--- a/apps/server/src/computer-use/shared/PARITY.sha256
+++ b/apps/server/src/computer-use/shared/PARITY.sha256
@@ -2,12 +2,12 @@
# A change to one must be ported to the other and this list updated in both.
36b49e5c1a90d4eb9b7d8c8fda207e6385c7ec0f8ba633e96f6726fc92ab2f53 contract.mjs
aef0973f520f56ab8ad2699c14428bb2dd7ddb4b00dff823e4598e64d0c4a35b contract.d.mts
-a147eeabed93c867fc12c8e205162dd4812a005de8176bbd005bc0cd8bc8bc5c policy.mjs
-57fec686eb98d9a80812355ea2c7bbe18cc8d0824934b5c8e9cdfc5669b0d757 policy.d.mts
+68cbfd310412988babd3fc772ac5bfd2842ec99d726ea7316e6d8b2a02fb5490 policy.mjs
+6d5cd2d2e8e1665806caca5b229ddb0513be130f78476e17d712e0f3647d46bd policy.d.mts
a96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs
5eac31868aedd3e1d1e71b445f9a55d01316da7397f27f6db11472453bee4588 image.d.mts
e47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjs
eabdf78c56733577d05aad490d9d7d5ece6c22e9b3ea9e6e9803d1a4a546fdd4 media.d.mts
-04bc6a36b9ce3409d0c7a5775b5fb6ea0797aa7ac304d83da249d19af9aca1be native-worker.mjs
-c7f42644d0e3b575e6896d705afeae88ec92b3f603dcc310854571fd6fa2f18d portable-host.mjs
+8543b410d31a33e700f4f6dbde3547ad0caa92421af387fe3e772a7444edfd60 native-worker.mjs
+7a55a915d4d725a4e992e7ef6191ba0ad1c851cdb5ee712ef79e0f7981a6a1c4 portable-host.mjs
2ed7be072df62d50e1b569e75b695d18ae0d0de6dabe5375bc494b859cf78710 portable-host.d.mts
diff --git a/apps/server/src/computer-use/shared/native-worker.mjs b/apps/server/src/computer-use/shared/native-worker.mjs
index 5721dd3e..31db5551 100644
--- a/apps/server/src/computer-use/shared/native-worker.mjs
+++ b/apps/server/src/computer-use/shared/native-worker.mjs
@@ -272,6 +272,53 @@ function windows() {
}
return out;
}
+ // The control under a screen point (MSAA, oleacc): its accessible name, role and protected state, for the
+ // host's per-act rules (risky targets, password fields). Never sent to the model. Null when unavailable.
+ let msaa = null;
+ const callPtr = (fn, proto, ...args) => (typeof koffi.call === "function" ? koffi.call(fn, proto, ...args) : koffi.decode(fn, koffi.pointer(proto))(...args));
+ const initMsaa = () => {
+ if (msaa !== null) return msaa;
+ try {
+ const ole = koffi.load("ole32.dll"), oleacc = koffi.load("oleacc.dll"), oleaut = koffi.load("oleaut32.dll");
+ f(ole, "int32_t __stdcall CoInitializeEx(void *reserved, uint32_t mode)")(null, 2);
+ koffi.struct("CU2_VARIANT", { vt: "uint16_t", r1: "uint16_t", r2: "uint16_t", r3: "uint16_t", lo: "int64_t", hi: "int64_t" });
+ msaa = {
+ fromPoint: f(oleacc, "int32_t __stdcall AccessibleObjectFromPoint(CU2_POINT point, _Out_ void **acc, _Out_ CU2_VARIANT *child)"),
+ length: f(oleaut, "uint32_t __stdcall SysStringLen(void *bstr)"),
+ free: f(oleaut, "void __stdcall SysFreeString(void *bstr)"),
+ name: koffi.proto("int32_t __stdcall CU2_ACC_NAME(void *self, CU2_VARIANT child, _Out_ void **name)"),
+ variant: koffi.proto("int32_t __stdcall CU2_ACC_VARIANT(void *self, CU2_VARIANT child, _Out_ CU2_VARIANT *out)"),
+ release: koffi.proto("uint32_t __stdcall CU2_ACC_RELEASE(void *self)"),
+ };
+ } catch { msaa = false; }
+ return msaa;
+ };
+ const low32 = (v) => Number(BigInt.asUintN(32, BigInt(v ?? 0)));
+ function targetAt(p) {
+ const m = initMsaa();
+ if (!m) return null;
+ const acc = [null], child = {};
+ try { if (m.fromPoint({ x: Math.round(p.x), y: Math.round(p.y) }, acc, child) < 0 || !acc[0]) return null; } catch { return null; }
+ // IAccessible vtable: IUnknown (0-2), IDispatch (3-6), get_accParent 7, ChildCount 8, Child 9, Name 10, Value 11, Description 12, Role 13, State 14.
+ const methods = koffi.decode(koffi.decode(acc[0], "void *"), "void *", 15);
+ try {
+ let name = "";
+ const bstr = [null];
+ if (callPtr(methods[10], m.name, acc[0], child, bstr) >= 0 && bstr[0]) {
+ try { const n = Math.min(m.length(bstr[0]), 400); name = Buffer.from(Uint16Array.from(koffi.decode(bstr[0], "uint16_t", n)).buffer).toString("utf16le"); }
+ finally { m.free(bstr[0]); }
+ }
+ const role = {}, state = {};
+ const hasRole = callPtr(methods[13], m.variant, acc[0], child, role) >= 0 && role.vt === 3;
+ const hasState = callPtr(methods[14], m.variant, acc[0], child, state) >= 0 && state.vt === 3;
+ // STATE_SYSTEM_PROTECTED (0x20000000): a password field.
+ return { name: name.trim().slice(0, 200), role: hasRole ? low32(role.lo) : null, protected: hasState && (low32(state.lo) & 0x20000000) !== 0 };
+ } catch {
+ return null;
+ } finally {
+ try { callPtr(methods[2], m.release, acc[0]); } catch { /* released with the process */ }
+ }
+ }
function screens() {
const out = [];
const cb = koffi.register((monitor) => {
@@ -436,7 +483,7 @@ function windows() {
} finally { release(); }
}
return {
- poll, window, screens, ancestors, available, release, startMonitoring, stopMonitoring, check, hit, capture, act,
+ poll, window, screens, ancestors, targetAt, available, release, startMonitoring, stopMonitoring, check, hit, capture, act,
cursor() { const p = {}; return cursorPos(p) ? { x: p.x, y: p.y } : null; },
list() { const result = []; const cb = koffi.register((id) => { try { const item = window(id); if (item.title) result.push(item); } catch { /* skip */ } return 1; }, koffi.pointer(enumProto)); try { enumWindows(cb, 0); } finally { koffi.unregister(cb); } return result; },
focus(expected) {
@@ -904,6 +951,7 @@ process.on("message", async (message) => {
else if (method === "capture") result = await backend.capture(params.window);
else if (method === "check") { backend.check(params.window, params.generation, params.deadline); result = {}; }
else if (method === "hit") result = { inside: (params.points ?? []).map((p) => backend.hit(params.window, p)) };
+ else if (method === "target") result = { targets: backend.targetAt ? (params.points ?? []).map((p) => backend.targetAt(p)) : null };
else if (method === "act") { await backend.act(params.window, params.generation, params.action, params.deadline); result = { generation }; }
else fail("unknown_method", "Unknown native operation.");
post({ id, result });
diff --git a/apps/server/src/computer-use/shared/policy.d.mts b/apps/server/src/computer-use/shared/policy.d.mts
index 250a6f1a..2a583ef3 100644
--- a/apps/server/src/computer-use/shared/policy.d.mts
+++ b/apps/server/src/computer-use/shared/policy.d.mts
@@ -56,7 +56,7 @@ export function looksLikeCredential(value: unknown): boolean;
export function riskyChord(action: unknown): "paste" | "close_quit_delete" | null;
export function documentOf(title: unknown, appName?: unknown): string;
-export type AskReasonCode = "careful" | "risky_intent" | "risky_target" | "secure_field" | "credential_text" | "paste" | "close_quit_delete" | "other_window" | "document_changed" | "after_takeover" | "submit_after_typing";
+export type AskReasonCode = "careful" | "risky_intent" | "risky_target" | "secure_field" | "credential_text" | "paste" | "close_quit_delete" | "other_window" | "document_changed" | "after_takeover" | "submit_after_typing" | "target_unknown";
export type ApprovalInput = {
mode?: PermissionMode | string;
action?: Action | Record<string, unknown>;
diff --git a/apps/server/src/computer-use/shared/policy.mjs b/apps/server/src/computer-use/shared/policy.mjs
index 7297fa94..914e92c1 100644
--- a/apps/server/src/computer-use/shared/policy.mjs
+++ b/apps/server/src/computer-use/shared/policy.mjs
@@ -365,9 +365,11 @@ export function actApproval(input = {}) {
const targetWords = riskyWords(target.text);
if (targetWords.length) reasons.push({ code: "risky_target", detail: targetWords.join(", ") });
}
- // No text the host could read at the target: a click or Enter right after typing in this window
- // may submit what was typed (a login, a message, a payment), so it asks.
- if (typeof target.text !== "string" && input.typed_before === true && (["click", "double_click"].includes(action.type) || (action.type === "key" && String(action.key).toLowerCase() === "enter"))) reasons.push({ code: "submit_after_typing" });
+ // No text the host could read at the target (X11; Windows when the control has no accessible name):
+ // a click could be "Pay" or "Delete account", so it asks; an Enter right after typing may submit what
+ // was typed (a login, a message, a payment), so it asks too.
+ if (typeof target.text !== "string" && ["click", "double_click", "right_click"].includes(action.type)) reasons.push({ code: "target_unknown" });
+ else if (typeof target.text !== "string" && input.typed_before === true && action.type === "key" && String(action.key).toLowerCase() === "enter") reasons.push({ code: "submit_after_typing" });
if (action.type === "type") {
if (target.secure === true) reasons.push({ code: "secure_field" });
if (looksLikeCredential(action.text)) reasons.push({ code: "credential_text" });
@@ -396,7 +398,8 @@ export const ASK_REASONS = Object.freeze({
other_window: "It would act in another window.",
document_changed: "The window now shows a different page or document",
after_takeover: "First action after you took over.",
- submit_after_typing: "It would click or press Enter right after typing, which may send what was typed.",
+ submit_after_typing: "It would press Enter right after typing, which may send what was typed.",
+ target_unknown: "The agent wants to click here; the app doesn't say what this control is.",
});
/**
diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjs
index 0fe3c51d..674ddd76 100644
--- a/apps/server/src/computer-use/shared/portable-host.mjs
+++ b/apps/server/src/computer-use/shared/portable-host.mjs
@@ -312,12 +312,19 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
}
/** The on-screen text at (or within 24 px of) the action's point, and whether it is in a secure field. Unknown text: undefined. */
- function targetAt(observed, action) {
+ async function targetAt(observed, action, current) {
const p = action.type === "drag" ? action.path.at(-1) : action.x !== undefined ? { x: action.x, y: action.y } : null;
if (!p) return {};
const inside = (r, pad) => p.x >= r.x - pad && p.x <= r.x + r.width + pad && p.y >= r.y - pad && p.y <= r.y + r.height + pad;
- const secure = (observed.secureFields ?? []).some((r) => inside(r, 0));
- if (!observed.texts) return secure ? { secure } : {};
+ let secure = (observed.secureFields ?? []).some((r) => inside(r, 0));
+ if (!observed.texts) {
+ // The control under the point, as the OS describes it (Windows MSAA); none elsewhere.
+ const answer = await native.call("target", { points: [mapPoint(p, observed.model, current.bounds).screen] }).catch(() => null);
+ const found = Array.isArray(answer?.targets) ? answer.targets[0] : null;
+ if (found?.protected) secure = true;
+ if (found && typeof found.name === "string" && found.name) return { text: found.name, ...(secure ? { secure } : {}) };
+ return secure ? { secure } : {};
+ }
const near = observed.texts.filter((t) => inside(t, 24)).sort((a, b) => (inside(a, 0) ? 0 : 1) - (inside(b, 0) ? 0 : 1)).slice(0, 4);
return { text: near.map((t) => t.text).join(" "), ...(secure ? { secure } : {}) };
}
@@ -364,7 +371,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
if (compare.stale) return finish({ ...common, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });
}
// Per-action approval by permission mode (host channel, never the engine's).
- const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: targetAt(observed, action), window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover, typed_before: s.typedIn === String(current.id) });
+ const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: await targetAt(observed, action, current), window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover, typed_before: s.typedIn === String(current.id) });
let approval = { asked: false };
if (decision.ask) {
// The same action with the same intent, allowed a moment ago (a retry after a fresh observation), is not asked again.
@@ -408,7 +415,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
s.lastOperation = Date.now();
// "Submit after typing" (Normal, when the host reads no text): typing arms it, a click or Enter after it asks.
if (action.type === "type") s.typedIn = String(current.id);
- else if (["click", "double_click"].includes(action.type) || (action.type === "key" && action.key === "Enter")) s.typedIn = null;
+ else if (["click", "double_click"].includes(action.type) || (action.type === "key" && String(action.key).toLowerCase() === "enter")) s.typedIn = null;
const cursorAfter = await native.call("cursor").catch(() => null);
const done = { ...common, approval, point, path, cursor_before: cursorBefore, cursor_after: cursorAfter, ok: true, may_have_acted: action.type !== "wait" };
if (!args.observe_after) return finish({ ...done, next: "computer_observe" });