A desktop coding agent with free access to frontier models.
TypeScript
fix: typing checks the focused field
ogsamratd41df7fa1705
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256index f96eef5b..4720e2c8 100644--- a/apps/server/src/computer-use/shared/PARITY.sha256+++ b/apps/server/src/computer-use/shared/PARITY.sha256@@ -8,6 +8,6 @@ a96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs5eac31868aedd3e1d1e71b445f9a55d01316da7397f27f6db11472453bee4588 image.d.mtse47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjseabdf78c56733577d05aad490d9d7d5ece6c22e9b3ea9e6e9803d1a4a546fdd4 media.d.mts-a6a6596c75df17c0aac6d7c740ae9661cae2b8899d12e20b5c2f37987cf9d4be native-worker.mjs-110b5f3aadaaec96b523fde230fa513d0fd75f2f7634b7336ac5b4ba975f18f4 portable-host.mjs+a0bd964a5d0c7c5e26b4ce4ed8e8300f80a0a3a9da5d7f49c36f6ac2c907e934 native-worker.mjs+9380ef9ababaae4f2defcef4eaeb3d6918fe24a0a3f710af3a428a371d8b5bbf portable-host.mjs641be80cebb04d353031392d8f53f45ead1e1beaf83f1e67d24945f0eda330bf portable-host.d.mtsdiff --git a/apps/server/src/computer-use/shared/native-worker.mjs b/apps/server/src/computer-use/shared/native-worker.mjsindex bec6109e..883294c9 100644--- a/apps/server/src/computer-use/shared/native-worker.mjs+++ b/apps/server/src/computer-use/shared/native-worker.mjs@@ -298,8 +298,11 @@ function windows() {function targetAt(p) {const m = initMsaa();if (!m) return null;- const acc = [null], child = {};+ const acc = [null];+ let child = {};try { if (m.fromPoint({ x: Math.round(p.x), y: Math.round(p.y) }, acc, child) < 0 || !acc[0]) return null; } catch { return null; }+ // Some servers (Chromium) leave the child VT_EMPTY: that is the object itself (VT_I4 CHILDID_SELF).+ if (child.vt !== 3) child = { vt: 3, r1: 0, r2: 0, r3: 0, lo: 0, hi: 0 };// IAccessible vtable: IUnknown (0-2), IDispatch (3-6), get_accParent 7, ChildCount 8, Child 9, Name 10, Value 11, Description 12, Role 13, State 14.const methods = koffi.decode(koffi.decode(acc[0], "void *"), "void *", 15);try {diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjsindex 7d8f8fd8..390ff774 100644--- a/apps/server/src/computer-use/shared/portable-host.mjs+++ b/apps/server/src/computer-use/shared/portable-host.mjs@@ -312,9 +312,20 @@ export async function createPortableHost({ native: firstNative, nativeFactory =}/** The on-screen text at (or within 24 px of) the action's point, and whether it is in a secure field. Unknown text: undefined. */- async function targetAt(observed, action, current) {+ /** Whether the control the last click focused (this window) is a secure field: { secure: true } or {}. */+ async function focusedTarget(observed, current, s) {+ const f = s?.focusPoint;+ if (!f || f.window !== String(current.id)) return {};+ const mx = ((f.screen.x - current.bounds.x) * observed.model.width) / current.bounds.width, my = ((f.screen.y - current.bounds.y) * observed.model.height) / current.bounds.height;+ if ((observed.secureFields ?? []).some((r) => mx >= r.x && mx <= r.x + r.width && my >= r.y && my <= r.y + r.height)) return { secure: true };+ const answer = await native.call("target", { points: [f.screen] }).catch(() => null);+ return Array.isArray(answer?.targets) && answer.targets[0]?.protected ? { secure: true } : {};+ }++ async function targetAt(observed, action, current, s) {const p = action.type === "drag" ? action.path.at(-1) : action.x !== undefined ? { x: action.x, y: action.y } : null;- if (!p) return {};+ // Typing has no point: it goes to the control the last click in this window focused.+ if (!p) return action.type === "type" ? focusedTarget(observed, current, s) : {};const inside = (r, pad) => p.x >= r.x - pad && p.x <= r.x + r.width + pad && p.y >= r.y - pad && p.y <= r.y + r.height + pad;let secure = (observed.secureFields ?? []).some((r) => inside(r, 0));if (!observed.texts) {@@ -371,7 +382,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =if (compare.stale) return finish({ ...common, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });}// Per-action approval by permission mode (host channel, never the engine's).- const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: await targetAt(observed, action, current), window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover, typed_before: s.typedIn === String(current.id) });+ const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: await targetAt(observed, action, current, s), window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover, typed_before: s.typedIn === String(current.id) });let approval = { asked: false };if (decision.ask) {// The same action with the same intent, allowed a moment ago (a retry after a fresh observation), is not asked again.@@ -414,6 +425,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =}s.lastOperation = Date.now();// "Submit after typing" (Normal, when the host reads no text): typing arms it, a click or Enter after it asks.+ if (["click", "double_click"].includes(action.type) && mapped.length === 1) s.focusPoint = { window: String(current.id), screen: mapped[0].screen };if (action.type === "type") s.typedIn = String(current.id);else if (["click", "double_click"].includes(action.type) || (action.type === "key" && String(action.key).toLowerCase() === "enter")) s.typedIn = null;const cursorAfter = await native.call("cursor").catch(() => null);