Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: typing checks the focused field

ogsamratd41df7fa1705
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256
index f96eef5b..4720e2c8 100644
--- a/apps/server/src/computer-use/shared/PARITY.sha256
+++ b/apps/server/src/computer-use/shared/PARITY.sha256
@@ -8,6 +8,6 @@ a96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs
5eac31868aedd3e1d1e71b445f9a55d01316da7397f27f6db11472453bee4588 image.d.mts
e47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjs
eabdf78c56733577d05aad490d9d7d5ece6c22e9b3ea9e6e9803d1a4a546fdd4 media.d.mts
-a6a6596c75df17c0aac6d7c740ae9661cae2b8899d12e20b5c2f37987cf9d4be native-worker.mjs
-110b5f3aadaaec96b523fde230fa513d0fd75f2f7634b7336ac5b4ba975f18f4 portable-host.mjs
+a0bd964a5d0c7c5e26b4ce4ed8e8300f80a0a3a9da5d7f49c36f6ac2c907e934 native-worker.mjs
+9380ef9ababaae4f2defcef4eaeb3d6918fe24a0a3f710af3a428a371d8b5bbf portable-host.mjs
641be80cebb04d353031392d8f53f45ead1e1beaf83f1e67d24945f0eda330bf portable-host.d.mts
diff --git a/apps/server/src/computer-use/shared/native-worker.mjs b/apps/server/src/computer-use/shared/native-worker.mjs
index bec6109e..883294c9 100644
--- a/apps/server/src/computer-use/shared/native-worker.mjs
+++ b/apps/server/src/computer-use/shared/native-worker.mjs
@@ -298,8 +298,11 @@ function windows() {
function targetAt(p) {
const m = initMsaa();
if (!m) return null;
- const acc = [null], child = {};
+ const acc = [null];
+ let child = {};
try { if (m.fromPoint({ x: Math.round(p.x), y: Math.round(p.y) }, acc, child) < 0 || !acc[0]) return null; } catch { return null; }
+ // Some servers (Chromium) leave the child VT_EMPTY: that is the object itself (VT_I4 CHILDID_SELF).
+ if (child.vt !== 3) child = { vt: 3, r1: 0, r2: 0, r3: 0, lo: 0, hi: 0 };
// IAccessible vtable: IUnknown (0-2), IDispatch (3-6), get_accParent 7, ChildCount 8, Child 9, Name 10, Value 11, Description 12, Role 13, State 14.
const methods = koffi.decode(koffi.decode(acc[0], "void *"), "void *", 15);
try {
diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjs
index 7d8f8fd8..390ff774 100644
--- a/apps/server/src/computer-use/shared/portable-host.mjs
+++ b/apps/server/src/computer-use/shared/portable-host.mjs
@@ -312,9 +312,20 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
}
/** The on-screen text at (or within 24 px of) the action's point, and whether it is in a secure field. Unknown text: undefined. */
- async function targetAt(observed, action, current) {
+ /** Whether the control the last click focused (this window) is a secure field: { secure: true } or {}. */
+ async function focusedTarget(observed, current, s) {
+ const f = s?.focusPoint;
+ if (!f || f.window !== String(current.id)) return {};
+ const mx = ((f.screen.x - current.bounds.x) * observed.model.width) / current.bounds.width, my = ((f.screen.y - current.bounds.y) * observed.model.height) / current.bounds.height;
+ if ((observed.secureFields ?? []).some((r) => mx >= r.x && mx <= r.x + r.width && my >= r.y && my <= r.y + r.height)) return { secure: true };
+ const answer = await native.call("target", { points: [f.screen] }).catch(() => null);
+ return Array.isArray(answer?.targets) && answer.targets[0]?.protected ? { secure: true } : {};
+ }
+
+ async function targetAt(observed, action, current, s) {
const p = action.type === "drag" ? action.path.at(-1) : action.x !== undefined ? { x: action.x, y: action.y } : null;
- if (!p) return {};
+ // Typing has no point: it goes to the control the last click in this window focused.
+ if (!p) return action.type === "type" ? focusedTarget(observed, current, s) : {};
const inside = (r, pad) => p.x >= r.x - pad && p.x <= r.x + r.width + pad && p.y >= r.y - pad && p.y <= r.y + r.height + pad;
let secure = (observed.secureFields ?? []).some((r) => inside(r, 0));
if (!observed.texts) {
@@ -371,7 +382,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
if (compare.stale) return finish({ ...common, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });
}
// Per-action approval by permission mode (host channel, never the engine's).
- const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: await targetAt(observed, action, current), window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover, typed_before: s.typedIn === String(current.id) });
+ const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: await targetAt(observed, action, current, s), window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover, typed_before: s.typedIn === String(current.id) });
let approval = { asked: false };
if (decision.ask) {
// The same action with the same intent, allowed a moment ago (a retry after a fresh observation), is not asked again.
@@ -414,6 +425,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
}
s.lastOperation = Date.now();
// "Submit after typing" (Normal, when the host reads no text): typing arms it, a click or Enter after it asks.
+ if (["click", "double_click"].includes(action.type) && mapped.length === 1) s.focusPoint = { window: String(current.id), screen: mapped[0].screen };
if (action.type === "type") s.typedIn = String(current.id);
else if (["click", "double_click"].includes(action.type) || (action.type === "key" && String(action.key).toLowerCase() === "enter")) s.typedIn = null;
const cursorAfter = await native.call("cursor").catch(() => null);