A desktop coding agent with free access to frontier models.
TypeScript
fix: computer use money rules, chat grant bounds and send/delete switch
ogsamratabdb47e83a3a
diff --git a/apps/server/src/computer-use/contract-policy.test.ts b/apps/server/src/computer-use/contract-policy.test.tsindex 1a4ec751..bb2e9355 100644--- a/apps/server/src/computer-use/contract-policy.test.ts+++ b/apps/server/src/computer-use/contract-policy.test.ts@@ -23,9 +23,9 @@ import {import {actApproval,ASK_REASONS,+ ASK_SEND_DELETE,computerUseSetting,deniedApp,- documentOf,looksLikeCredential,modelAllowed,nestedAgent,@@ -226,6 +226,14 @@ describe("policy: approvals per permission mode", () => {["checkout target, Enter", { action: { type: "key", key: "Enter" }, target: { text: "Checkout" } }, ["risky_target"]],["subscribe target", { target: { text: "Subscribe for $9.99/month" } }, ["risky_target"]],["donate intent", { intent: "Donate 5 dollars" }, ["risky_intent"]],+ ["transfer intent", { intent: "Transfer $500 to savings" }, ["risky_intent"]],+ ["send money target", { target: { text: "Send $20" } }, ["risky_target"]],+ ["reorder target", { target: { text: "Reorder" } }, ["risky_target"]],+ ["book now target", { target: { text: "Book now" } }, ["risky_target"]],+ ["add funds target", { target: { text: "Add funds" } }, ["risky_target"]],+ ["Space on a focused pay button", { action: { type: "key", key: "Space" }, target: { text: "Pay" } }, ["risky_target"]],+ ["no text at the point, checkout page (X11)", { target: {}, window: { ...window, title: "Checkout - Example Shop" } }, ["risky_page"]],+ ["no text at the point, cart page, Enter (X11)", { action: { type: "key", key: "Enter" }, target: {}, window: { ...window, title: "Your cart" } }, ["risky_page"]],// (b) typing a password: secure field, credential text, or an intent that says so.["secure field", { action: { type: "type", text: "hunter22" }, target: { secure: true } }, ["secure_field"]],["credential text", { action: { type: "type", text: "ghp_" + "a1B2".repeat(9) } }, ["credential_text"]],@@ -249,6 +257,11 @@ describe("policy: approvals per permission mode", () => {["after takeover", { after_takeover: true }, []],["sort order", { intent: "Sort in order to find the row" }, []],["order history", { target: { text: "Order history" } }, []],+ ["price next to a plain button", { target: { text: "Add to cart $19.99" } }, []],+ ["no text at the point, mail page (X11)", { target: {}, window: { ...window, title: "Inbox - Mail" } }, []],+ ["send target (switch off)", { target: { text: "Send" } }, []],+ ["empty trash target (switch off)", { target: { text: "Empty Trash" } }, []],+ ["shift+delete (switch off)", { action: { type: "key", key: "Delete", modifiers: ["shift"] } }, []],["plain typing", { action: { type: "type", text: "Meeting notes for Tuesday" }, intent: "Type the notes" }, []],["move over a pay button", { action: { type: "move", x: 1, y: 1 }, target: { text: "Pay" } }, []],// Kept: an act outside the approved window (a session never sends one).@@ -268,15 +281,29 @@ describe("policy: approvals per permission mode", () => {expect(riskyWords("uninstall the plugin then post a reply")).toEqual([]);expect(riskyWords("buy the plan, then place order")).toEqual(["buy", "place order"]);expect(riskyWords("track my order")).toEqual([]);+ expect(riskyWords("Send $20 to Bob")).toEqual(["amount"]);+ expect(riskyWords("Transfer 500 EUR")).toEqual(["transfer", "amount"]);expect(actApproval({ ...plain, intent: "Buy it" }).reasons[0]).toEqual({ code: "risky_intent", detail: "buy" });for (const code of ["careful", "risky_intent", "risky_target", "secure_field", "credential_text", "credential_intent", "other_window"]) expect(typeof ASK_REASONS[code as keyof typeof ASK_REASONS]).toBe("string");- expect(Object.keys(ASK_REASONS).sort()).toEqual(["careful", "credential_intent", "credential_text", "other_window", "risky_intent", "risky_target", "secure_field"]);+ expect(Object.keys(ASK_REASONS).sort()).toEqual(["careful", "credential_intent", "credential_text", "delete_chord", "other_window", "risky_intent", "risky_page", "risky_target", "secure_field", "send_delete_target"]);expect(looksLikeCredential("AKIAABCDEFGHIJKLMNOP")).toBe(true);expect(looksLikeCredential("password=Sup3rS3cret!")).toBe(true);expect(looksLikeCredential("Xk9#mP2$vL7qR4@nT8wZ")).toBe(true);expect(looksLikeCredential("https://example.com/some/long/path?q=1")).toBe(false);expect(looksLikeCredential("the quick brown fox jumps")).toBe(false);- expect(documentOf("● main.ts - Code", "Code")).toBe("main.ts");+ });++ test("the send/delete switch (off by default): OS-read send and permanent-delete targets and delete chords ask; intents never", () => {+ expect(ASK_SEND_DELETE).toBe(false);+ const on = { sendDelete: true };+ for (const text of ["Send", "Post", "Publish", "Forward", "Reply all", "Delete permanently", "Empty Trash", "Delete account"]) expect(actApproval({ ...plain, target: { text } }, on).reasons.map((r) => r.code)).toEqual(["send_delete_target"]);+ expect(actApproval({ ...plain, action: { type: "key", key: "Enter" }, target: { text: "Send" } }, on).ask).toBe(true);+ for (const chord of [{ key: "Delete", modifiers: ["shift"] }, { key: "Backspace", modifiers: ["meta"] }]) expect(actApproval({ ...plain, action: { type: "key", ...chord } }, on).reasons).toEqual([{ code: "delete_chord" }]);+ expect(actApproval({ ...plain, intent: "Send the email and delete the draft" }, on).ask).toBe(false);+ expect(actApproval({ ...plain, target: {} }, on).ask).toBe(false);+ expect(actApproval({ ...plain, target: { text: "Move to Trash" } }, on).ask).toBe(false);+ expect(actApproval({ ...plain, target: { text: "Sender" } }, on).ask).toBe(false);+ expect(actApproval({ ...plain, mode: "auto", target: { text: "Send" } }, on).ask).toBe(false);});test("Allow for this chat: until the chat ends by default; the server's value is an upper bound", () => {diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.tsindex 48cf89f2..ff2126f2 100644--- a/apps/server/src/computer-use/portable-host.test.ts+++ b/apps/server/src/computer-use/portable-host.test.ts@@ -176,6 +176,30 @@ describe("portable host: MCP surface and consent", () => {expect(b.opens[0]?.chat_seconds).toBe(600);await hb.close();shift = 0;+ // The server lowers its bound after the grant: the next reuse is checked against the new one.+ const lowered = rig("normal", { decision: "chat" });+ const hl = await lowered.hostPromise;+ const cl = hl.connect();+ const g = await call(cl, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "Write" });+ await call(cl, "computer_close_session", { session_id: g.state.session_id });+ lowered.settings.limits = { chat_approval_seconds: 1800 };+ shift = 1801_000;+ await call(cl, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "Write" });+ expect(lowered.opens.length).toBe(2);+ await hl.close();+ shift = 0;+ // The person's Stop and the server's switch clear the grant.+ for (const by of ["person", "server"] as const) {+ const st = rig("normal", { decision: "chat" });+ const hs = await st.hostPromise;+ const cs = hs.connect();+ const x = await call(cs, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "Write" });+ if (by === "person") await hs.control({ session_id: String(x.state.session_id), action: "stop" });+ else await hs.stopAll("server");+ await call(cs, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "Write" });+ expect(st.opens.length).toBe(2);+ await hs.close();+ }// Another chat asks again; no chat known: no grant.for (const chats of [["ses_1", "ses_2"], [null, null]]) {const d = rig("normal", { decision: "chat", chats });@@ -447,6 +471,24 @@ describe("portable host: the person's own press answers a question", () => {});});+describe("portable host: Enter and Space read the focused control; Tab forgets it", () => {+ test("Enter on a focused pay button asks; after Tab it is unknown and runs", async () => {+ const r = rig("normal");+ r.native.state.texts = [{ x: 300, y: 200, width: 80, height: 30, text: "Pay now" }];+ const { host, c, session, obs } = await opened(r);+ const click = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Focus the button", action: { type: "click", x: 300, y: 200 } });+ expect(click.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["risky_target"] } });+ const enter = await call(c, "computer_act", { session_id: session, observation_id: (click.state.observation as { id: string }).id, request_id: "b", intent: "Press it", action: { type: "key", key: "Enter" } });+ expect(enter.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["risky_target"] } });+ const tab = await call(c, "computer_act", { session_id: session, observation_id: (enter.state.observation as { id: string }).id, request_id: "c", intent: "Next control", action: { type: "key", key: "Tab" } });+ expect(tab.state).toMatchObject({ ok: true, approval: { asked: false } });+ const after = await call(c, "computer_act", { session_id: session, observation_id: (tab.state.observation as { id: string }).id, request_id: "d", intent: "Press it", action: { type: "key", key: "Enter" } });+ expect(after.state).toMatchObject({ ok: true, approval: { asked: false } });+ expect(r.acts.length).toBe(2);+ await host.close();+ });+});+describe("portable host: the control under the point from the worker (Windows MSAA)", () => {test("a named control decides (pay target, protected field); no name runs", async () => {const r = rig("normal");diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256index 0773b91f..8c10862f 100644--- a/apps/server/src/computer-use/shared/PARITY.sha256+++ b/apps/server/src/computer-use/shared/PARITY.sha256@@ -2,12 +2,12 @@# A change to one must be ported to the other and this list updated in both.59abb350a935c9462f3ea2e863396cdc785634dd2907f2f8947d87e9b30ec209 contract.mjsaef0973f520f56ab8ad2699c14428bb2dd7ddb4b00dff823e4598e64d0c4a35b contract.d.mts-43810f0b2d2fba8b51878ab895d6475a98b2582f05c918995c554a6aa148503b policy.mjs-88d60f32b244cde8a1c1328ee7ac13d1e61c7ef1d0f036fffcc780c0ef7c1e6e policy.d.mts+f5b379f5c8270acf2bbbd319206d9217d7bd03fc0e18102c3d0334cd5396f9e6 policy.mjs+8bb6597a3dd4d2b5b12ffcda52d344144c5f512e1d79b78cdbf6c743e17cd3d2 policy.d.mtsa96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs5eac31868aedd3e1d1e71b445f9a55d01316da7397f27f6db11472453bee4588 image.d.mtse47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjseabdf78c56733577d05aad490d9d7d5ece6c22e9b3ea9e6e9803d1a4a546fdd4 media.d.mtsa0bd964a5d0c7c5e26b4ce4ed8e8300f80a0a3a9da5d7f49c36f6ac2c907e934 native-worker.mjs-329272889926cfcff24fa7e4c0188edc0f63b34e9bb3a73a0cc4361403bf7b0a portable-host.mjs+2ca377ffc46df9ae4530a73bad1ce5b49f66d8b985b375f18f7a0d3e9f128e66 portable-host.mjs641be80cebb04d353031392d8f53f45ead1e1beaf83f1e67d24945f0eda330bf portable-host.d.mtsdiff --git a/apps/server/src/computer-use/shared/policy.d.mts b/apps/server/src/computer-use/shared/policy.d.mtsindex 2474dadc..df4fda36 100644--- a/apps/server/src/computer-use/shared/policy.d.mts+++ b/apps/server/src/computer-use/shared/policy.d.mts@@ -53,18 +53,18 @@ export function nestedAgent(options?: { env?: Record<string, string | undefined>export const RISKY_WORDS: readonly string[];export function riskyWords(value: unknown): string[];export function looksLikeCredential(value: unknown): boolean;-export function documentOf(title: unknown, appName?: unknown): string;+export const ASK_SEND_DELETE: boolean;-export type AskReasonCode = "careful" | "risky_intent" | "risky_target" | "secure_field" | "credential_text" | "credential_intent" | "other_window";+export type AskReasonCode = "careful" | "risky_intent" | "risky_target" | "secure_field" | "credential_text" | "credential_intent" | "other_window" | "risky_page" | "send_delete_target" | "delete_chord";export type ApprovalInput = {mode?: PermissionMode | string;action?: Action | Record<string, unknown>;intent?: string;target?: { text?: string; secure?: boolean };- window?: { id?: string | number; app_id?: string };+ window?: { id?: string | number; app_id?: string; title?: string };approved?: { window_id?: string | number; app_id?: string };};-export function actApproval(input?: ApprovalInput): { ask: boolean; reasons: Array<{ code: AskReasonCode; detail?: string }> };+export function actApproval(input?: ApprovalInput, options?: { sendDelete?: boolean }): { ask: boolean; reasons: Array<{ code: AskReasonCode; detail?: string }> };export const ASK_REASONS: Readonly<Record<AskReasonCode, string>>;export const SCREEN_BASH_RULES: Readonly<Record<string, "ask">>;export function computerPermissionRules(): Record<string, "allow">;diff --git a/apps/server/src/computer-use/shared/policy.mjs b/apps/server/src/computer-use/shared/policy.mjsindex e0474aa3..cfc0b6c5 100644--- a/apps/server/src/computer-use/shared/policy.mjs+++ b/apps/server/src/computer-use/shared/policy.mjs@@ -258,13 +258,17 @@ export function nestedAgent({ env = process.env, ancestors = [] } = {}) {* paying, buying and ordering (money leaving the person's account).*/export const RISKY_WORDS = Object.freeze([- "pay", "payment", "pay now", "buy", "buy now", "purchase", "checkout", "check out", "order", "place order", "order now",- "subscribe", "donate", "send money", "transfer money", "wire",+ "pay", "payment", "pay now", "buy", "buy now", "buy again", "purchase", "checkout", "check out", "order", "place order", "order now", "reorder", "1-click",+ "subscribe", "upgrade", "renew", "donate", "tip", "bid", "place bid", "book", "book now", "reserve", "rent", "top up", "top-up", "add funds",+ "send money", "transfer", "wire",]);const RISKY_RE = new RegExp(`(^|[^\\p{L}\\p{N}])(${[...RISKY_WORDS].sort((a, b) => b.length - a.length).map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/ /g, "\\s+")).join("|")})(?=$|[^\\p{L}\\p{N}])`, "iu");-const BENIGN = /\b(in\s+order\s+(to|for)|sort\s+order|order\s+by|reorder|ordered\s+list|order\s+(history|status|details|number)|track\s+(my\s+|your\s+|the\s+)?order)\b/gi;+const BENIGN = /\b(in\s+order\s+(to|for)|sort\s+order|order\s+by|ordered\s+list|order\s+(history|status|details|number)|track\s+(my\s+|your\s+|the\s+)?order)\b/gi;+// An amount of money with a verb that moves it ("Send $20", "Confirm 500 EUR"): paying too.+const AMOUNT = /[$€£₹¥]\s?\d|\b\d[\d,.]*\s?(usd|eur|gbp|inr|jpy|dollars?|euros?|rupees?|pounds?)\b/i;+const MOVES_MONEY = /\b(send|transfer|give|confirm|complete|submit|request|withdraw|deposit|pay|tip|donate)\b/i;-/** The paying/buying/ordering words found in `value` (intent or on-screen text), lower case. */+/** The paying/buying/ordering words found in `value` (intent or on-screen text), lower case ("amount": money with a verb that moves it). */export function riskyWords(value) {if (typeof value !== "string" || !value) return [];const textValue = value.replace(BENIGN, " ");@@ -272,9 +276,22 @@ export function riskyWords(value) {const re = new RegExp(RISKY_RE.source, "giu");let m;while ((m = re.exec(textValue)) !== null) found.add(m[2].toLowerCase().replace(/\s+/g, " "));+ if (AMOUNT.test(textValue) && MOVES_MONEY.test(textValue)) found.add("amount");return [...found];}+/** Window titles (a browser's page title) that say the page takes money: hosts that read no text at the point (X11) ask there. */+const PAYMENT_PAGE = /\b(cart|basket|billing|payment|checkout|check\s*out)\b/i;++/**+ * The judge's narrower variant (send and delete), OFF by default: when true, Normal also asks when the text+ * the OS read at a click/Enter target says Send/Post/Publish/Forward/Reply all or a permanent delete, and on+ * the Shift+Delete and Cmd+Backspace chords. Only OS-read text counts, never the agent's intent.+ */+export const ASK_SEND_DELETE = false;+const SEND_TARGET = /(^|[^\p{L}\p{N}])(send|post|publish|forward|reply\s+all)(?=$|[^\p{L}\p{N}])/iu;+const DELETE_TARGET = /\b(delete\s+(permanently|forever|account)|permanently\s+delete|empty\s+(the\s+)?(trash|bin|recycle\s+bin))\b/i;+/** An intent that says the typed text is a password, code or key (hosts that cannot see secure fields, X11). */const SECRET_INTENT = /\b(pass(word|code|phrase)s?|pins?|one[-\s]?time\s+(code|password)|otp|2fa|mfa|verification\s+code|security\s+code|secret|api[-\s]?key|access\s+key|token|credentials?|cvv|cvc|card\s+number)\b/i;@@ -314,49 +331,53 @@ export function looksLikeCredential(value) {return classes >= 3 && entropy(word) >= 3.8 && !/^https?:\/\//i.test(word) && !/^[\w.+-]+@[\w-]+\.[\w.]+$/.test(word);}-/** A window title without unsaved-change markers and the app name suffix, for "same document" checks. */-export function documentOf(title, appName) {- let t = String(title ?? "").trim().replace(/^[*•●◦]\s*/, "").replace(/\s*[*•●◦]$/, "").replace(/\s+\(modified\)$/i, "");- const app = String(appName ?? "").trim();- if (app) for (const sep of [" - ", " — ", " – ", " | "]) if (t.toLowerCase().endsWith((sep + app).toLowerCase())) t = t.slice(0, -(sep + app).length);- return t.replace(/\s+/g, " ").trim().toLowerCase();-}-/*** Whether one computer_act asks the person first, by permission mode:* - careful: every act asks;- * - normal: only paying, buying or ordering (the intent, or the text at a- * click or Enter target) and typing a password (a secure field, text that+ * - normal: only paying, buying or ordering (the intent, the text at a click+ * or Enter/Space target, or, where the host reads no text there, a page+ * title that says so) and typing a password (a secure field, text that* looks like a credential, or an intent that says so); plus an act outside* the approved window, which a session never does (acts go to the approved- * window, other apps need their own open consent);+ * window, other apps need their own open consent). With ASK_SEND_DELETE+ * (or options.sendDelete) also OS-read send/permanent-delete targets and+ * the Shift+Delete / Cmd+Backspace chords;* - auto: never.* In every mode the open-session consent, the deny list, hit-testing, the* indicator and Stop still apply.* `input`: { mode, action, intent, target: { text, secure }, window: { id,- * app_id }, approved: { window_id, app_id } } (target.text undefined: the- * host reads no text there).+ * app_id, title }, approved: { window_id, app_id } } (target.text undefined:+ * the host reads no text there). `options`: { sendDelete }.* Returns { ask, reasons: [{ code, detail? }] }.*/-export function actApproval(input = {}) {+export function actApproval(input = {}, options = {}) {const mode = PERMISSION_MODES.includes(input.mode) ? input.mode : "normal";if (mode === "auto") return { ask: false, reasons: [] };if (mode === "careful") return { ask: true, reasons: [{ code: "careful" }] };+ const sendDelete = options.sendDelete ?? ASK_SEND_DELETE;const reasons = [];const action = isRecord(input.action) ? input.action : {};const target = isRecord(input.target) ? input.target : {};+ const window = isRecord(input.window) ? input.window : null;+ const key = action.type === "key" ? String(action.key ?? "").toLowerCase() : "";+ const mods = new Set(Array.isArray(action.modifiers) ? action.modifiers : []);const intentWords = riskyWords(input.intent);if (intentWords.length) reasons.push({ code: "risky_intent", detail: intentWords.join(", ") });- if (["click", "double_click", "right_click", "drag"].includes(action.type) || (action.type === "key" && String(action.key).toLowerCase() === "enter")) {+ // Clicks, and Enter or Space on the focused control (no modifiers).+ const presses = ["click", "double_click", "right_click", "drag"].includes(action.type) || ((key === "enter" || key === "space") && mods.size === 0);+ if (presses) {const targetWords = riskyWords(target.text);if (targetWords.length) reasons.push({ code: "risky_target", detail: targetWords.join(", ") });+ // No text the host could read there (X11): the page title decides whether this may be a payment page.+ else if (typeof target.text !== "string" && action.type !== "drag" && window && (riskyWords(window.title).length || PAYMENT_PAGE.test(String(window.title ?? "")))) reasons.push({ code: "risky_page", detail: String(window.title ?? "").slice(0, 120) });+ if (sendDelete && typeof target.text === "string" && (SEND_TARGET.test(target.text) || DELETE_TARGET.test(target.text))) reasons.push({ code: "send_delete_target", detail: target.text.slice(0, 80) });}+ if (sendDelete && action.type === "key" && ((mods.has("shift") && key === "delete") || (mods.has("meta") && (key === "backspace" || key === "delete")))) reasons.push({ code: "delete_chord" });if (action.type === "type") {if (target.secure === true) reasons.push({ code: "secure_field" });if (looksLikeCredential(action.text)) reasons.push({ code: "credential_text" });else if (target.secure !== true && SECRET_INTENT.test(String(input.intent ?? ""))) reasons.push({ code: "credential_intent" });}- const window = isRecord(input.window) ? input.window : null;const approved = isRecord(input.approved) ? input.approved : null;if (window && approved && ((approved.app_id && window.app_id && approved.app_id !== window.app_id) || (approved.window_id !== undefined && window.id !== undefined && String(approved.window_id) !== String(window.id)))) reasons.push({ code: "other_window" });return { ask: reasons.length > 0, reasons };@@ -367,6 +388,9 @@ export const ASK_REASONS = Object.freeze({careful: "Careful mode asks before every action.",risky_intent: "The agent says it will pay, buy or order something",risky_target: "The target looks like a button that pays, buys or orders",+ risky_page: "The page looks like a checkout or payment page",+ send_delete_target: "The target looks like a button that sends, posts or deletes for good",+ delete_chord: "The shortcut deletes for good.",secure_field: "It would type into a password field.",credential_text: "The text looks like a password or key.",credential_intent: "The agent says it is typing a password, code or key.",diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjsindex ad2a23ba..2d796654 100644--- a/apps/server/src/computer-use/shared/portable-host.mjs+++ b/apps/server/src/computer-use/shared/portable-host.mjs@@ -311,21 +311,8 @@ export async function createPortableHost({ native: firstNative, nativeFactory =return mcpResult(observeResult({ session_id: s.id, observation: obs.ref, window: windowRef(w), screen: s.screen, scale: w.bounds.width / obs.model.width, dpi: w.dpi, monitor_id: w.monitor, cursor: await native.call("cursor").catch(() => null), os, expires_in_ms: l.observation_ttl_seconds * 1000, actions_left: Math.max(0, l.actions_per_session - s.actions), seconds_left: Math.max(0, Math.ceil((s.expiresAt - Date.now()) / 1000)) }), includeImage ? { data: obs.bytes, mime: obs.mime } : null);}- /** The on-screen text at (or within 24 px of) the action's point, and whether it is in a secure field. Unknown text: undefined. */- /** Whether the control the last click focused (this window) is a secure field: { secure: true } or {}. */- async function focusedTarget(observed, current, s) {- const f = s?.focusPoint;- if (!f || f.window !== String(current.id)) return {};- const mx = ((f.screen.x - current.bounds.x) * observed.model.width) / current.bounds.width, my = ((f.screen.y - current.bounds.y) * observed.model.height) / current.bounds.height;- if ((observed.secureFields ?? []).some((r) => mx >= r.x && mx <= r.x + r.width && my >= r.y && my <= r.y + r.height)) return { secure: true };- const answer = await native.call("target", { points: [f.screen] }).catch(() => null);- return Array.isArray(answer?.targets) && answer.targets[0]?.protected ? { secure: true } : {};- }-- async function targetAt(observed, action, current, s) {- const p = action.type === "drag" ? action.path.at(-1) : action.x !== undefined ? { x: action.x, y: action.y } : null;- // Typing has no point: it goes to the control the last click in this window focused.- if (!p) return action.type === "type" ? focusedTarget(observed, current, s) : {};+ /** The on-screen text at (or within 24 px of) a point of the image, and whether it is in a secure field. Unknown text: undefined. */+ async function targetAtPoint(observed, current, p) {const inside = (r, pad) => p.x >= r.x - pad && p.x <= r.x + r.width + pad && p.y >= r.y - pad && p.y <= r.y + r.height + pad;let secure = (observed.secureFields ?? []).some((r) => inside(r, 0));if (!observed.texts) {@@ -340,6 +327,21 @@ export async function createPortableHost({ native: firstNative, nativeFactory =return { text: near.map((t) => t.text).join(" "), ...(secure ? { secure } : {}) };}+ async function targetAt(observed, action, current, s) {+ const p = action.type === "drag" ? action.path.at(-1) : action.x !== undefined ? { x: action.x, y: action.y } : null;+ if (p) return targetAtPoint(observed, current, p);+ // Typing, Enter and Space have no point: they go to the control the last click in this window focused+ // (none after a Tab moved the focus: unknown).+ const key = action.type === "key" ? String(action.key ?? "").toLowerCase() : "";+ if (action.type !== "type" && key !== "enter" && key !== "space") return {};+ const f = s?.focusPoint;+ if (!f || f.window !== String(current.id)) return {};+ const model = { x: ((f.screen.x - current.bounds.x) * observed.model.width) / current.bounds.width, y: ((f.screen.y - current.bounds.y) * observed.model.height) / current.bounds.height };+ const found = await targetAtPoint(observed, current, model);+ // Typing: only whether the field is secure (its label is not what is typed).+ return action.type === "type" ? (found.secure ? { secure: true } : {}) : found;+ }+async function actTool(s, args) {const l = limits();const started = Date.now();@@ -382,7 +384,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =if (compare.stale) return finish({ ...common, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });}// Per-action approval by permission mode (host channel, never the engine's).- const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: await targetAt(observed, action, current, s), window: { id: String(current.id), app_id: current.appId }, approved: s.approved });+ const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: await targetAt(observed, action, current, s), window: { id: String(current.id), app_id: current.appId, title: current.title }, approved: s.approved });let approval = { asked: false };if (decision.ask) {// The same action with the same intent, allowed a moment ago (a retry after a fresh observation), is not asked again.@@ -424,6 +426,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =s.lastOperation = Date.now();// Typing goes to the control the last click focused: its secure-field check reads this point.if (["click", "double_click"].includes(action.type) && mapped.length === 1) s.focusPoint = { window: String(current.id), screen: mapped[0].screen };+ else if (action.type === "key" && String(action.key).toLowerCase() === "tab") s.focusPoint = null;const cursorAfter = await native.call("cursor").catch(() => null);const done = { ...common, approval, point, path, cursor_before: cursorBefore, cursor_after: cursorAfter, ok: true, may_have_acted: action.type !== "wait" };if (!args.observe_after) return finish({ ...done, next: "computer_observe" });@@ -491,8 +494,10 @@ export async function createPortableHost({ native: firstNative, nativeFactory =update();const grantKey = `${s.chat ?? ""}\0${preferred.appId}\0${s.mode}`;const grant = s.chat ? grants.get(grantKey) : null;+ // The server's bound now (it may have been lowered since): 0 = until the chat ends.+ const fresh = grant && (l.chat_approval_seconds === 0 || Date.now() - grant.at < l.chat_approval_seconds * 1000);let answer;- if (grant && grant.until > Date.now()) answer = { decision: "chat", window_id: String(preferred.id), reused: true };+ if (fresh) answer = { decision: "chat", window_id: String(preferred.id), reused: true };else {const pendingAnswer = new Promise((resolve) => { s.pendingApproval = { resolve }; });const request = { session_id: s.id, request_id: randomUUID(), app: { app_id: preferred.appId, app_name: preferred.appName, bundle_or_exe: preferred.executable, pid: preferred.pid }, windows: windows.map((w) => ({ id: String(w.id), title: w.title })), window_id: String(preferred.id), mode: s.mode, purpose: s.purpose, os, seconds: l.session_seconds, chat_seconds: l.chat_approval_seconds };@@ -525,9 +530,10 @@ export async function createPortableHost({ native: firstNative, nativeFactory =}s.approval = answer.decision;// "Allow for this chat": this app, this chat, until the chat ends (chat_approval_seconds 0), or for at most the- // server's chat_approval_seconds. Kept in memory only, keyed to the chat: never another chat, never a restart.- // A call no chat can be named for gets no grant (it works as "Allow once").- if (answer.decision === "chat" && !answer.reused && s.chat) grants.set(grantKey, { until: l.chat_approval_seconds > 0 ? Date.now() + l.chat_approval_seconds * 1000 : Infinity });+ // server's chat_approval_seconds (checked again at each reuse). Kept in memory only, keyed to the chat: never+ // another chat, never a restart; the person's Stop and the server's switch clear it. A call no chat can be+ // named for gets no grant (it works as "Allow once").+ if (answer.decision === "chat" && !answer.reused && s.chat) grants.set(grantKey, { at: Date.now() });s.phase = "working"; s.lastOperation = Date.now();s.expiresAt = Date.now() + l.session_seconds * 1000;s.approved = { window_id: String(s.window.id), app_id: s.window.appId };@@ -637,7 +643,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =async function control({ session_id, action }) {const s = sessions.get(session_id);if (!s) throw new Error("This computer use session has ended.");- if (action === "stop") return stop(s, "stopped", "person");+ if (action === "stop") { grants.clear(); return stop(s, "stopped", "person"); }if (action === "takeover") return pause(s, "person", "You took over. Choose Continue when you are ready.");if (action === "hide" || action === "show") { s.hidden = action === "hide"; update(); return undefined; }if (action !== "continue" || s.phase !== "paused") throw new Error("This action is unavailable.");@@ -656,6 +662,8 @@ export async function createPortableHost({ native: firstNative, nativeFactory =/** Ends every session (server switch off, app quitting). */async function stopAll(by = "server", event = "stopped") {+ // Stop by the person or the server: "Allow for this chat" is asked again.+ if (by === "person" || by === "server") grants.clear();for (const s of [...sessions.values()]) await stop(s, event, by);}