A desktop coding agent with free access to frontier models.
TypeScript
fix: ask before clicks on unnamed controls
ogsamrat2c6d2734c3e6
diff --git a/apps/server/src/computer-use/contract-policy.test.ts b/apps/server/src/computer-use/contract-policy.test.tsindex 6d7f8182..6a98fce0 100644--- a/apps/server/src/computer-use/contract-policy.test.ts+++ b/apps/server/src/computer-use/contract-policy.test.ts@@ -250,10 +250,12 @@ describe("policy: approvals per permission mode", () => {expect(documentOf("● main.ts - Code", "Code")).toBe("main.ts");});- test("Normal, no readable text at the target: a click or Enter right after typing asks", () => {- expect(actApproval({ ...plain, target: {}, typed_before: true }).reasons).toEqual([{ code: "submit_after_typing" }]);- expect(actApproval({ ...plain, action: { type: "key", key: "Enter" }, target: {}, typed_before: true }).ask).toBe(true);- expect(actApproval({ ...plain, target: {}, typed_before: false }).ask).toBe(false);+ test("Normal, no readable text at the target: a click asks (target unknown); Enter right after typing asks", () => {+ expect(actApproval({ ...plain, target: {} }).reasons).toEqual([{ code: "target_unknown" }]);+ expect(actApproval({ ...plain, action: { type: "right_click", x: 1, y: 1 }, target: {} }).ask).toBe(true);+ expect(actApproval({ ...plain, action: { type: "key", key: "Enter" }, target: {}, typed_before: true }).reasons).toEqual([{ code: "submit_after_typing" }]);+ expect(actApproval({ ...plain, action: { type: "key", key: "enter" }, target: {}, typed_before: false }).ask).toBe(false);+ expect(actApproval({ ...plain, action: { type: "move", x: 1, y: 1 }, target: {} }).ask).toBe(false);// With text the host read there, the target rules decide instead.expect(actApproval({ ...plain, target: { text: "Bold" }, typed_before: true }).ask).toBe(false);expect(actApproval({ ...plain, mode: "auto", target: {}, typed_before: true }).ask).toBe(false);diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.tsindex fe3d77ef..1b2f5391 100644--- a/apps/server/src/computer-use/portable-host.test.ts+++ b/apps/server/src/computer-use/portable-host.test.ts@@ -200,6 +200,7 @@ describe("portable host: observe and act", () => {test("Normal: plain acts run, risky ones ask", async () => {const r = rig("normal");+ r.native.state.texts = [{ x: 0, y: 0, width: 100, height: 100, text: "Notes" }];const { host, c, session, obs } = await opened(r);const plain = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Put the cursor at the end", action: { type: "click", x: 5, y: 5 } });expect(plain.state).toMatchObject({ ok: true, approval: { asked: false } });@@ -259,14 +260,15 @@ describe("portable host: observe and act", () => {const secure = await call(c, "computer_act", { session_id: session, observation_id: (sign.state.observation as { id: string }).id, request_id: "b", intent: "Fill in the field", action: { type: "type", text: "abc" } });expect(secure.state).toMatchObject({ ok: true, approval: { asked: false } });await host.close();- // A host that reads no text (X11, Windows): type, then a click asks.+ // A host that reads no text (X11): every click asks (the app doesn't say what it is); typing does not.const x = rig("normal");const o = await opened(x);const typed = await call(o.c, "computer_act", { session_id: o.session, observation_id: o.obs.state.observation?.id, request_id: "t", intent: "Type the name", action: { type: "type", text: "Ada" } });expect(typed.state).toMatchObject({ ok: true, approval: { asked: false } });const click = await call(o.c, "computer_act", { session_id: o.session, observation_id: (typed.state.observation as { id: string }).id, request_id: "k", intent: "Go to the next field", action: { type: "click", x: 400, y: 300 } });- expect(click.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["submit_after_typing"] } });- const again = await call(o.c, "computer_act", { session_id: o.session, observation_id: (click.state.observation as { id: string }).id, request_id: "k2", intent: "Click again", action: { type: "click", x: 400, y: 300 } });+ expect(click.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["target_unknown"] } });+ // The same click on the same image within 30 s (a retry) is not asked again.+ const again = await call(o.c, "computer_act", { session_id: o.session, observation_id: (click.state.observation as { id: string }).id, request_id: "k2", intent: "Go to the next field", action: { type: "click", x: 400, y: 300 } });expect(again.state).toMatchObject({ ok: true, approval: { asked: false } });expect(await o.host.physicalInput()).toBe(true);x.native.state.synthetic = true;@@ -284,7 +286,7 @@ describe("portable host: observe and act", () => {await host.control({ session_id: session, action: "continue" });const next = await call(c, "computer_observe", { session_id: session });const act = await call(c, "computer_act", { session_id: session, observation_id: next.state.observation?.id, request_id: "b", intent: "Click in the text", action: { type: "click", x: 5, y: 5 } });- expect(act.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["after_takeover"] } });+ expect(act.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["target_unknown", "after_takeover"] } });expect(r.events.map((e) => `${e.event}:${e.by}`)).toEqual(["opened:agent", "approved:person", "paused:person", "continued:person"]);await host.control({ session_id: session, action: "stop" });expect(r.events.at(-1)).toMatchObject({ event: "stopped", by: "person", app_id: "/usr/bin/gedit", app_name: "gedit", bundle_or_exe: "/usr/bin/gedit", mode: "control", approval: "once", os: "linux", screen: { id: "0", width: 3840, height: 2160 } });@@ -367,6 +369,25 @@ describe("portable host: titles that follow the agent's own typing, and a worker});});+describe("portable host: the control under the point from the worker (Windows MSAA)", () => {+ test("a named control decides (risky target, protected field); no name falls back to target_unknown", async () => {+ const r = rig("normal");+ let next: { name: string; role: number | null; protected: boolean } | null = { name: "Pay now", role: 43, protected: false };+ const call0 = r.native.call.bind(r.native);+ r.native.call = async (method: string, params: Record<string, unknown> = {}) => (method === "target" ? { targets: [next] } : call0(method, params));+ const { host, c, session, obs } = await opened(r);+ const pay = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "p", intent: "Press the button", action: { type: "click", x: 10, y: 10 } });+ expect(pay.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["risky_target"] } });+ next = { name: "Bold", role: 43, protected: false };+ const bold = await call(c, "computer_act", { session_id: session, observation_id: (pay.state.observation as { id: string }).id, request_id: "b", intent: "Make it bold", action: { type: "click", x: 20, y: 10 } });+ expect(bold.state).toMatchObject({ ok: true, approval: { asked: false } });+ next = { name: "", role: 42, protected: false };+ const blank = await call(c, "computer_act", { session_id: session, observation_id: (bold.state.observation as { id: string }).id, request_id: "n", intent: "Click there", action: { type: "click", x: 30, y: 10 } });+ expect(blank.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["target_unknown"] } });+ await host.close();+ });+});+describe("portable host on Windows: the agent's ancestors come from the worker", () => {test("an IDE the agent runs in is denied as an ancestor when the app passed none", async () => {const native = fakeNative();diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256index 47262c32..940d1332 100644--- a/apps/server/src/computer-use/shared/PARITY.sha256+++ b/apps/server/src/computer-use/shared/PARITY.sha256@@ -2,12 +2,12 @@# A change to one must be ported to the other and this list updated in both.36b49e5c1a90d4eb9b7d8c8fda207e6385c7ec0f8ba633e96f6726fc92ab2f53 contract.mjsaef0973f520f56ab8ad2699c14428bb2dd7ddb4b00dff823e4598e64d0c4a35b contract.d.mts-a147eeabed93c867fc12c8e205162dd4812a005de8176bbd005bc0cd8bc8bc5c policy.mjs-57fec686eb98d9a80812355ea2c7bbe18cc8d0824934b5c8e9cdfc5669b0d757 policy.d.mts+68cbfd310412988babd3fc772ac5bfd2842ec99d726ea7316e6d8b2a02fb5490 policy.mjs+6d5cd2d2e8e1665806caca5b229ddb0513be130f78476e17d712e0f3647d46bd policy.d.mtsa96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs5eac31868aedd3e1d1e71b445f9a55d01316da7397f27f6db11472453bee4588 image.d.mtse47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjseabdf78c56733577d05aad490d9d7d5ece6c22e9b3ea9e6e9803d1a4a546fdd4 media.d.mts-04bc6a36b9ce3409d0c7a5775b5fb6ea0797aa7ac304d83da249d19af9aca1be native-worker.mjs-c7f42644d0e3b575e6896d705afeae88ec92b3f603dcc310854571fd6fa2f18d portable-host.mjs+8543b410d31a33e700f4f6dbde3547ad0caa92421af387fe3e772a7444edfd60 native-worker.mjs+7a55a915d4d725a4e992e7ef6191ba0ad1c851cdb5ee712ef79e0f7981a6a1c4 portable-host.mjs2ed7be072df62d50e1b569e75b695d18ae0d0de6dabe5375bc494b859cf78710 portable-host.d.mtsdiff --git a/apps/server/src/computer-use/shared/native-worker.mjs b/apps/server/src/computer-use/shared/native-worker.mjsindex 5721dd3e..31db5551 100644--- a/apps/server/src/computer-use/shared/native-worker.mjs+++ b/apps/server/src/computer-use/shared/native-worker.mjs@@ -272,6 +272,53 @@ function windows() {}return out;}+ // The control under a screen point (MSAA, oleacc): its accessible name, role and protected state, for the+ // host's per-act rules (risky targets, password fields). Never sent to the model. Null when unavailable.+ let msaa = null;+ const callPtr = (fn, proto, ...args) => (typeof koffi.call === "function" ? koffi.call(fn, proto, ...args) : koffi.decode(fn, koffi.pointer(proto))(...args));+ const initMsaa = () => {+ if (msaa !== null) return msaa;+ try {+ const ole = koffi.load("ole32.dll"), oleacc = koffi.load("oleacc.dll"), oleaut = koffi.load("oleaut32.dll");+ f(ole, "int32_t __stdcall CoInitializeEx(void *reserved, uint32_t mode)")(null, 2);+ koffi.struct("CU2_VARIANT", { vt: "uint16_t", r1: "uint16_t", r2: "uint16_t", r3: "uint16_t", lo: "int64_t", hi: "int64_t" });+ msaa = {+ fromPoint: f(oleacc, "int32_t __stdcall AccessibleObjectFromPoint(CU2_POINT point, _Out_ void **acc, _Out_ CU2_VARIANT *child)"),+ length: f(oleaut, "uint32_t __stdcall SysStringLen(void *bstr)"),+ free: f(oleaut, "void __stdcall SysFreeString(void *bstr)"),+ name: koffi.proto("int32_t __stdcall CU2_ACC_NAME(void *self, CU2_VARIANT child, _Out_ void **name)"),+ variant: koffi.proto("int32_t __stdcall CU2_ACC_VARIANT(void *self, CU2_VARIANT child, _Out_ CU2_VARIANT *out)"),+ release: koffi.proto("uint32_t __stdcall CU2_ACC_RELEASE(void *self)"),+ };+ } catch { msaa = false; }+ return msaa;+ };+ const low32 = (v) => Number(BigInt.asUintN(32, BigInt(v ?? 0)));+ function targetAt(p) {+ const m = initMsaa();+ if (!m) return null;+ const acc = [null], child = {};+ try { if (m.fromPoint({ x: Math.round(p.x), y: Math.round(p.y) }, acc, child) < 0 || !acc[0]) return null; } catch { return null; }+ // IAccessible vtable: IUnknown (0-2), IDispatch (3-6), get_accParent 7, ChildCount 8, Child 9, Name 10, Value 11, Description 12, Role 13, State 14.+ const methods = koffi.decode(koffi.decode(acc[0], "void *"), "void *", 15);+ try {+ let name = "";+ const bstr = [null];+ if (callPtr(methods[10], m.name, acc[0], child, bstr) >= 0 && bstr[0]) {+ try { const n = Math.min(m.length(bstr[0]), 400); name = Buffer.from(Uint16Array.from(koffi.decode(bstr[0], "uint16_t", n)).buffer).toString("utf16le"); }+ finally { m.free(bstr[0]); }+ }+ const role = {}, state = {};+ const hasRole = callPtr(methods[13], m.variant, acc[0], child, role) >= 0 && role.vt === 3;+ const hasState = callPtr(methods[14], m.variant, acc[0], child, state) >= 0 && state.vt === 3;+ // STATE_SYSTEM_PROTECTED (0x20000000): a password field.+ return { name: name.trim().slice(0, 200), role: hasRole ? low32(role.lo) : null, protected: hasState && (low32(state.lo) & 0x20000000) !== 0 };+ } catch {+ return null;+ } finally {+ try { callPtr(methods[2], m.release, acc[0]); } catch { /* released with the process */ }+ }+ }function screens() {const out = [];const cb = koffi.register((monitor) => {@@ -436,7 +483,7 @@ function windows() {} finally { release(); }}return {- poll, window, screens, ancestors, available, release, startMonitoring, stopMonitoring, check, hit, capture, act,+ poll, window, screens, ancestors, targetAt, available, release, startMonitoring, stopMonitoring, check, hit, capture, act,cursor() { const p = {}; return cursorPos(p) ? { x: p.x, y: p.y } : null; },list() { const result = []; const cb = koffi.register((id) => { try { const item = window(id); if (item.title) result.push(item); } catch { /* skip */ } return 1; }, koffi.pointer(enumProto)); try { enumWindows(cb, 0); } finally { koffi.unregister(cb); } return result; },focus(expected) {@@ -904,6 +951,7 @@ process.on("message", async (message) => {else if (method === "capture") result = await backend.capture(params.window);else if (method === "check") { backend.check(params.window, params.generation, params.deadline); result = {}; }else if (method === "hit") result = { inside: (params.points ?? []).map((p) => backend.hit(params.window, p)) };+ else if (method === "target") result = { targets: backend.targetAt ? (params.points ?? []).map((p) => backend.targetAt(p)) : null };else if (method === "act") { await backend.act(params.window, params.generation, params.action, params.deadline); result = { generation }; }else fail("unknown_method", "Unknown native operation.");post({ id, result });diff --git a/apps/server/src/computer-use/shared/policy.d.mts b/apps/server/src/computer-use/shared/policy.d.mtsindex 250a6f1a..2a583ef3 100644--- a/apps/server/src/computer-use/shared/policy.d.mts+++ b/apps/server/src/computer-use/shared/policy.d.mts@@ -56,7 +56,7 @@ export function looksLikeCredential(value: unknown): boolean;export function riskyChord(action: unknown): "paste" | "close_quit_delete" | null;export function documentOf(title: unknown, appName?: unknown): string;-export type AskReasonCode = "careful" | "risky_intent" | "risky_target" | "secure_field" | "credential_text" | "paste" | "close_quit_delete" | "other_window" | "document_changed" | "after_takeover" | "submit_after_typing";+export type AskReasonCode = "careful" | "risky_intent" | "risky_target" | "secure_field" | "credential_text" | "paste" | "close_quit_delete" | "other_window" | "document_changed" | "after_takeover" | "submit_after_typing" | "target_unknown";export type ApprovalInput = {mode?: PermissionMode | string;action?: Action | Record<string, unknown>;diff --git a/apps/server/src/computer-use/shared/policy.mjs b/apps/server/src/computer-use/shared/policy.mjsindex 7297fa94..914e92c1 100644--- a/apps/server/src/computer-use/shared/policy.mjs+++ b/apps/server/src/computer-use/shared/policy.mjs@@ -365,9 +365,11 @@ export function actApproval(input = {}) {const targetWords = riskyWords(target.text);if (targetWords.length) reasons.push({ code: "risky_target", detail: targetWords.join(", ") });}- // No text the host could read at the target: a click or Enter right after typing in this window- // may submit what was typed (a login, a message, a payment), so it asks.- if (typeof target.text !== "string" && input.typed_before === true && (["click", "double_click"].includes(action.type) || (action.type === "key" && String(action.key).toLowerCase() === "enter"))) reasons.push({ code: "submit_after_typing" });+ // No text the host could read at the target (X11; Windows when the control has no accessible name):+ // a click could be "Pay" or "Delete account", so it asks; an Enter right after typing may submit what+ // was typed (a login, a message, a payment), so it asks too.+ if (typeof target.text !== "string" && ["click", "double_click", "right_click"].includes(action.type)) reasons.push({ code: "target_unknown" });+ else if (typeof target.text !== "string" && input.typed_before === true && action.type === "key" && String(action.key).toLowerCase() === "enter") reasons.push({ code: "submit_after_typing" });if (action.type === "type") {if (target.secure === true) reasons.push({ code: "secure_field" });if (looksLikeCredential(action.text)) reasons.push({ code: "credential_text" });@@ -396,7 +398,8 @@ export const ASK_REASONS = Object.freeze({other_window: "It would act in another window.",document_changed: "The window now shows a different page or document",after_takeover: "First action after you took over.",- submit_after_typing: "It would click or press Enter right after typing, which may send what was typed.",+ submit_after_typing: "It would press Enter right after typing, which may send what was typed.",+ target_unknown: "The agent wants to click here; the app doesn't say what this control is.",});/**diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjsindex 0fe3c51d..674ddd76 100644--- a/apps/server/src/computer-use/shared/portable-host.mjs+++ b/apps/server/src/computer-use/shared/portable-host.mjs@@ -312,12 +312,19 @@ export async function createPortableHost({ native: firstNative, nativeFactory =}/** The on-screen text at (or within 24 px of) the action's point, and whether it is in a secure field. Unknown text: undefined. */- function targetAt(observed, action) {+ async function targetAt(observed, action, current) {const p = action.type === "drag" ? action.path.at(-1) : action.x !== undefined ? { x: action.x, y: action.y } : null;if (!p) return {};const inside = (r, pad) => p.x >= r.x - pad && p.x <= r.x + r.width + pad && p.y >= r.y - pad && p.y <= r.y + r.height + pad;- const secure = (observed.secureFields ?? []).some((r) => inside(r, 0));- if (!observed.texts) return secure ? { secure } : {};+ let secure = (observed.secureFields ?? []).some((r) => inside(r, 0));+ if (!observed.texts) {+ // The control under the point, as the OS describes it (Windows MSAA); none elsewhere.+ const answer = await native.call("target", { points: [mapPoint(p, observed.model, current.bounds).screen] }).catch(() => null);+ const found = Array.isArray(answer?.targets) ? answer.targets[0] : null;+ if (found?.protected) secure = true;+ if (found && typeof found.name === "string" && found.name) return { text: found.name, ...(secure ? { secure } : {}) };+ return secure ? { secure } : {};+ }const near = observed.texts.filter((t) => inside(t, 24)).sort((a, b) => (inside(a, 0) ? 0 : 1) - (inside(b, 0) ? 0 : 1)).slice(0, 4);return { text: near.map((t) => t.text).join(" "), ...(secure ? { secure } : {}) };}@@ -364,7 +371,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =if (compare.stale) return finish({ ...common, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });}// Per-action approval by permission mode (host channel, never the engine's).- const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: targetAt(observed, action), window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover, typed_before: s.typedIn === String(current.id) });+ const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: await targetAt(observed, action, current), window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover, typed_before: s.typedIn === String(current.id) });let approval = { asked: false };if (decision.ask) {// The same action with the same intent, allowed a moment ago (a retry after a fresh observation), is not asked again.@@ -408,7 +415,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =s.lastOperation = Date.now();// "Submit after typing" (Normal, when the host reads no text): typing arms it, a click or Enter after it asks.if (action.type === "type") s.typedIn = String(current.id);- else if (["click", "double_click"].includes(action.type) || (action.type === "key" && action.key === "Enter")) s.typedIn = null;+ else if (["click", "double_click"].includes(action.type) || (action.type === "key" && String(action.key).toLowerCase() === "enter")) s.typedIn = null;const cursorAfter = await native.call("cursor").catch(() => null);const done = { ...common, approval, point, path, cursor_before: cursorBefore, cursor_after: cursorAfter, ok: true, may_have_acted: action.type !== "wait" };if (!args.observe_after) return finish({ ...done, next: "computer_observe" });