Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: computer use money rules, chat grant bounds and send/delete switch

ogsamratabdb47e83a3a
diff --git a/apps/server/src/computer-use/contract-policy.test.ts b/apps/server/src/computer-use/contract-policy.test.ts
index 1a4ec751..bb2e9355 100644
--- a/apps/server/src/computer-use/contract-policy.test.ts
+++ b/apps/server/src/computer-use/contract-policy.test.ts
@@ -23,9 +23,9 @@ import {
import {
actApproval,
ASK_REASONS,
+ ASK_SEND_DELETE,
computerUseSetting,
deniedApp,
- documentOf,
looksLikeCredential,
modelAllowed,
nestedAgent,
@@ -226,6 +226,14 @@ describe("policy: approvals per permission mode", () => {
["checkout target, Enter", { action: { type: "key", key: "Enter" }, target: { text: "Checkout" } }, ["risky_target"]],
["subscribe target", { target: { text: "Subscribe for $9.99/month" } }, ["risky_target"]],
["donate intent", { intent: "Donate 5 dollars" }, ["risky_intent"]],
+ ["transfer intent", { intent: "Transfer $500 to savings" }, ["risky_intent"]],
+ ["send money target", { target: { text: "Send $20" } }, ["risky_target"]],
+ ["reorder target", { target: { text: "Reorder" } }, ["risky_target"]],
+ ["book now target", { target: { text: "Book now" } }, ["risky_target"]],
+ ["add funds target", { target: { text: "Add funds" } }, ["risky_target"]],
+ ["Space on a focused pay button", { action: { type: "key", key: "Space" }, target: { text: "Pay" } }, ["risky_target"]],
+ ["no text at the point, checkout page (X11)", { target: {}, window: { ...window, title: "Checkout - Example Shop" } }, ["risky_page"]],
+ ["no text at the point, cart page, Enter (X11)", { action: { type: "key", key: "Enter" }, target: {}, window: { ...window, title: "Your cart" } }, ["risky_page"]],
// (b) typing a password: secure field, credential text, or an intent that says so.
["secure field", { action: { type: "type", text: "hunter22" }, target: { secure: true } }, ["secure_field"]],
["credential text", { action: { type: "type", text: "ghp_" + "a1B2".repeat(9) } }, ["credential_text"]],
@@ -249,6 +257,11 @@ describe("policy: approvals per permission mode", () => {
["after takeover", { after_takeover: true }, []],
["sort order", { intent: "Sort in order to find the row" }, []],
["order history", { target: { text: "Order history" } }, []],
+ ["price next to a plain button", { target: { text: "Add to cart $19.99" } }, []],
+ ["no text at the point, mail page (X11)", { target: {}, window: { ...window, title: "Inbox - Mail" } }, []],
+ ["send target (switch off)", { target: { text: "Send" } }, []],
+ ["empty trash target (switch off)", { target: { text: "Empty Trash" } }, []],
+ ["shift+delete (switch off)", { action: { type: "key", key: "Delete", modifiers: ["shift"] } }, []],
["plain typing", { action: { type: "type", text: "Meeting notes for Tuesday" }, intent: "Type the notes" }, []],
["move over a pay button", { action: { type: "move", x: 1, y: 1 }, target: { text: "Pay" } }, []],
// Kept: an act outside the approved window (a session never sends one).
@@ -268,15 +281,29 @@ describe("policy: approvals per permission mode", () => {
expect(riskyWords("uninstall the plugin then post a reply")).toEqual([]);
expect(riskyWords("buy the plan, then place order")).toEqual(["buy", "place order"]);
expect(riskyWords("track my order")).toEqual([]);
+ expect(riskyWords("Send $20 to Bob")).toEqual(["amount"]);
+ expect(riskyWords("Transfer 500 EUR")).toEqual(["transfer", "amount"]);
expect(actApproval({ ...plain, intent: "Buy it" }).reasons[0]).toEqual({ code: "risky_intent", detail: "buy" });
for (const code of ["careful", "risky_intent", "risky_target", "secure_field", "credential_text", "credential_intent", "other_window"]) expect(typeof ASK_REASONS[code as keyof typeof ASK_REASONS]).toBe("string");
- expect(Object.keys(ASK_REASONS).sort()).toEqual(["careful", "credential_intent", "credential_text", "other_window", "risky_intent", "risky_target", "secure_field"]);
+ expect(Object.keys(ASK_REASONS).sort()).toEqual(["careful", "credential_intent", "credential_text", "delete_chord", "other_window", "risky_intent", "risky_page", "risky_target", "secure_field", "send_delete_target"]);
expect(looksLikeCredential("AKIAABCDEFGHIJKLMNOP")).toBe(true);
expect(looksLikeCredential("password=Sup3rS3cret!")).toBe(true);
expect(looksLikeCredential("Xk9#mP2$vL7qR4@nT8wZ")).toBe(true);
expect(looksLikeCredential("https://example.com/some/long/path?q=1")).toBe(false);
expect(looksLikeCredential("the quick brown fox jumps")).toBe(false);
- expect(documentOf("● main.ts - Code", "Code")).toBe("main.ts");
+ });
+
+ test("the send/delete switch (off by default): OS-read send and permanent-delete targets and delete chords ask; intents never", () => {
+ expect(ASK_SEND_DELETE).toBe(false);
+ const on = { sendDelete: true };
+ for (const text of ["Send", "Post", "Publish", "Forward", "Reply all", "Delete permanently", "Empty Trash", "Delete account"]) expect(actApproval({ ...plain, target: { text } }, on).reasons.map((r) => r.code)).toEqual(["send_delete_target"]);
+ expect(actApproval({ ...plain, action: { type: "key", key: "Enter" }, target: { text: "Send" } }, on).ask).toBe(true);
+ for (const chord of [{ key: "Delete", modifiers: ["shift"] }, { key: "Backspace", modifiers: ["meta"] }]) expect(actApproval({ ...plain, action: { type: "key", ...chord } }, on).reasons).toEqual([{ code: "delete_chord" }]);
+ expect(actApproval({ ...plain, intent: "Send the email and delete the draft" }, on).ask).toBe(false);
+ expect(actApproval({ ...plain, target: {} }, on).ask).toBe(false);
+ expect(actApproval({ ...plain, target: { text: "Move to Trash" } }, on).ask).toBe(false);
+ expect(actApproval({ ...plain, target: { text: "Sender" } }, on).ask).toBe(false);
+ expect(actApproval({ ...plain, mode: "auto", target: { text: "Send" } }, on).ask).toBe(false);
});
test("Allow for this chat: until the chat ends by default; the server's value is an upper bound", () => {
diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.ts
index 48cf89f2..ff2126f2 100644
--- a/apps/server/src/computer-use/portable-host.test.ts
+++ b/apps/server/src/computer-use/portable-host.test.ts
@@ -176,6 +176,30 @@ describe("portable host: MCP surface and consent", () => {
expect(b.opens[0]?.chat_seconds).toBe(600);
await hb.close();
shift = 0;
+ // The server lowers its bound after the grant: the next reuse is checked against the new one.
+ const lowered = rig("normal", { decision: "chat" });
+ const hl = await lowered.hostPromise;
+ const cl = hl.connect();
+ const g = await call(cl, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "Write" });
+ await call(cl, "computer_close_session", { session_id: g.state.session_id });
+ lowered.settings.limits = { chat_approval_seconds: 1800 };
+ shift = 1801_000;
+ await call(cl, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "Write" });
+ expect(lowered.opens.length).toBe(2);
+ await hl.close();
+ shift = 0;
+ // The person's Stop and the server's switch clear the grant.
+ for (const by of ["person", "server"] as const) {
+ const st = rig("normal", { decision: "chat" });
+ const hs = await st.hostPromise;
+ const cs = hs.connect();
+ const x = await call(cs, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "Write" });
+ if (by === "person") await hs.control({ session_id: String(x.state.session_id), action: "stop" });
+ else await hs.stopAll("server");
+ await call(cs, "computer_open_session", { app_id: "/usr/bin/gedit", mode: "control", purpose: "Write" });
+ expect(st.opens.length).toBe(2);
+ await hs.close();
+ }
// Another chat asks again; no chat known: no grant.
for (const chats of [["ses_1", "ses_2"], [null, null]]) {
const d = rig("normal", { decision: "chat", chats });
@@ -447,6 +471,24 @@ describe("portable host: the person's own press answers a question", () => {
});
});
+describe("portable host: Enter and Space read the focused control; Tab forgets it", () => {
+ test("Enter on a focused pay button asks; after Tab it is unknown and runs", async () => {
+ const r = rig("normal");
+ r.native.state.texts = [{ x: 300, y: 200, width: 80, height: 30, text: "Pay now" }];
+ const { host, c, session, obs } = await opened(r);
+ const click = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Focus the button", action: { type: "click", x: 300, y: 200 } });
+ expect(click.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["risky_target"] } });
+ const enter = await call(c, "computer_act", { session_id: session, observation_id: (click.state.observation as { id: string }).id, request_id: "b", intent: "Press it", action: { type: "key", key: "Enter" } });
+ expect(enter.state).toMatchObject({ ok: true, approval: { asked: true, reasons: ["risky_target"] } });
+ const tab = await call(c, "computer_act", { session_id: session, observation_id: (enter.state.observation as { id: string }).id, request_id: "c", intent: "Next control", action: { type: "key", key: "Tab" } });
+ expect(tab.state).toMatchObject({ ok: true, approval: { asked: false } });
+ const after = await call(c, "computer_act", { session_id: session, observation_id: (tab.state.observation as { id: string }).id, request_id: "d", intent: "Press it", action: { type: "key", key: "Enter" } });
+ expect(after.state).toMatchObject({ ok: true, approval: { asked: false } });
+ expect(r.acts.length).toBe(2);
+ await host.close();
+ });
+});
+
describe("portable host: the control under the point from the worker (Windows MSAA)", () => {
test("a named control decides (pay target, protected field); no name runs", async () => {
const r = rig("normal");
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256
index 0773b91f..8c10862f 100644
--- a/apps/server/src/computer-use/shared/PARITY.sha256
+++ b/apps/server/src/computer-use/shared/PARITY.sha256
@@ -2,12 +2,12 @@
# A change to one must be ported to the other and this list updated in both.
59abb350a935c9462f3ea2e863396cdc785634dd2907f2f8947d87e9b30ec209 contract.mjs
aef0973f520f56ab8ad2699c14428bb2dd7ddb4b00dff823e4598e64d0c4a35b contract.d.mts
-43810f0b2d2fba8b51878ab895d6475a98b2582f05c918995c554a6aa148503b policy.mjs
-88d60f32b244cde8a1c1328ee7ac13d1e61c7ef1d0f036fffcc780c0ef7c1e6e policy.d.mts
+f5b379f5c8270acf2bbbd319206d9217d7bd03fc0e18102c3d0334cd5396f9e6 policy.mjs
+8bb6597a3dd4d2b5b12ffcda52d344144c5f512e1d79b78cdbf6c743e17cd3d2 policy.d.mts
a96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs
5eac31868aedd3e1d1e71b445f9a55d01316da7397f27f6db11472453bee4588 image.d.mts
e47c9ff67b9fce2ec12307737e973ce22551719c482b87f601569bd4f8a77e17 media.mjs
eabdf78c56733577d05aad490d9d7d5ece6c22e9b3ea9e6e9803d1a4a546fdd4 media.d.mts
a0bd964a5d0c7c5e26b4ce4ed8e8300f80a0a3a9da5d7f49c36f6ac2c907e934 native-worker.mjs
-329272889926cfcff24fa7e4c0188edc0f63b34e9bb3a73a0cc4361403bf7b0a portable-host.mjs
+2ca377ffc46df9ae4530a73bad1ce5b49f66d8b985b375f18f7a0d3e9f128e66 portable-host.mjs
641be80cebb04d353031392d8f53f45ead1e1beaf83f1e67d24945f0eda330bf portable-host.d.mts
diff --git a/apps/server/src/computer-use/shared/policy.d.mts b/apps/server/src/computer-use/shared/policy.d.mts
index 2474dadc..df4fda36 100644
--- a/apps/server/src/computer-use/shared/policy.d.mts
+++ b/apps/server/src/computer-use/shared/policy.d.mts
@@ -53,18 +53,18 @@ export function nestedAgent(options?: { env?: Record<string, string | undefined>
export const RISKY_WORDS: readonly string[];
export function riskyWords(value: unknown): string[];
export function looksLikeCredential(value: unknown): boolean;
-export function documentOf(title: unknown, appName?: unknown): string;
+export const ASK_SEND_DELETE: boolean;
-export type AskReasonCode = "careful" | "risky_intent" | "risky_target" | "secure_field" | "credential_text" | "credential_intent" | "other_window";
+export type AskReasonCode = "careful" | "risky_intent" | "risky_target" | "secure_field" | "credential_text" | "credential_intent" | "other_window" | "risky_page" | "send_delete_target" | "delete_chord";
export type ApprovalInput = {
mode?: PermissionMode | string;
action?: Action | Record<string, unknown>;
intent?: string;
target?: { text?: string; secure?: boolean };
- window?: { id?: string | number; app_id?: string };
+ window?: { id?: string | number; app_id?: string; title?: string };
approved?: { window_id?: string | number; app_id?: string };
};
-export function actApproval(input?: ApprovalInput): { ask: boolean; reasons: Array<{ code: AskReasonCode; detail?: string }> };
+export function actApproval(input?: ApprovalInput, options?: { sendDelete?: boolean }): { ask: boolean; reasons: Array<{ code: AskReasonCode; detail?: string }> };
export const ASK_REASONS: Readonly<Record<AskReasonCode, string>>;
export const SCREEN_BASH_RULES: Readonly<Record<string, "ask">>;
export function computerPermissionRules(): Record<string, "allow">;
diff --git a/apps/server/src/computer-use/shared/policy.mjs b/apps/server/src/computer-use/shared/policy.mjs
index e0474aa3..cfc0b6c5 100644
--- a/apps/server/src/computer-use/shared/policy.mjs
+++ b/apps/server/src/computer-use/shared/policy.mjs
@@ -258,13 +258,17 @@ export function nestedAgent({ env = process.env, ancestors = [] } = {}) {
* paying, buying and ordering (money leaving the person's account).
*/
export const RISKY_WORDS = Object.freeze([
- "pay", "payment", "pay now", "buy", "buy now", "purchase", "checkout", "check out", "order", "place order", "order now",
- "subscribe", "donate", "send money", "transfer money", "wire",
+ "pay", "payment", "pay now", "buy", "buy now", "buy again", "purchase", "checkout", "check out", "order", "place order", "order now", "reorder", "1-click",
+ "subscribe", "upgrade", "renew", "donate", "tip", "bid", "place bid", "book", "book now", "reserve", "rent", "top up", "top-up", "add funds",
+ "send money", "transfer", "wire",
]);
const RISKY_RE = new RegExp(`(^|[^\\p{L}\\p{N}])(${[...RISKY_WORDS].sort((a, b) => b.length - a.length).map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/ /g, "\\s+")).join("|")})(?=$|[^\\p{L}\\p{N}])`, "iu");
-const BENIGN = /\b(in\s+order\s+(to|for)|sort\s+order|order\s+by|reorder|ordered\s+list|order\s+(history|status|details|number)|track\s+(my\s+|your\s+|the\s+)?order)\b/gi;
+const BENIGN = /\b(in\s+order\s+(to|for)|sort\s+order|order\s+by|ordered\s+list|order\s+(history|status|details|number)|track\s+(my\s+|your\s+|the\s+)?order)\b/gi;
+// An amount of money with a verb that moves it ("Send $20", "Confirm 500 EUR"): paying too.
+const AMOUNT = /[$€£₹¥]\s?\d|\b\d[\d,.]*\s?(usd|eur|gbp|inr|jpy|dollars?|euros?|rupees?|pounds?)\b/i;
+const MOVES_MONEY = /\b(send|transfer|give|confirm|complete|submit|request|withdraw|deposit|pay|tip|donate)\b/i;
-/** The paying/buying/ordering words found in `value` (intent or on-screen text), lower case. */
+/** The paying/buying/ordering words found in `value` (intent or on-screen text), lower case ("amount": money with a verb that moves it). */
export function riskyWords(value) {
if (typeof value !== "string" || !value) return [];
const textValue = value.replace(BENIGN, " ");
@@ -272,9 +276,22 @@ export function riskyWords(value) {
const re = new RegExp(RISKY_RE.source, "giu");
let m;
while ((m = re.exec(textValue)) !== null) found.add(m[2].toLowerCase().replace(/\s+/g, " "));
+ if (AMOUNT.test(textValue) && MOVES_MONEY.test(textValue)) found.add("amount");
return [...found];
}
+/** Window titles (a browser's page title) that say the page takes money: hosts that read no text at the point (X11) ask there. */
+const PAYMENT_PAGE = /\b(cart|basket|billing|payment|checkout|check\s*out)\b/i;
+
+/**
+ * The judge's narrower variant (send and delete), OFF by default: when true, Normal also asks when the text
+ * the OS read at a click/Enter target says Send/Post/Publish/Forward/Reply all or a permanent delete, and on
+ * the Shift+Delete and Cmd+Backspace chords. Only OS-read text counts, never the agent's intent.
+ */
+export const ASK_SEND_DELETE = false;
+const SEND_TARGET = /(^|[^\p{L}\p{N}])(send|post|publish|forward|reply\s+all)(?=$|[^\p{L}\p{N}])/iu;
+const DELETE_TARGET = /\b(delete\s+(permanently|forever|account)|permanently\s+delete|empty\s+(the\s+)?(trash|bin|recycle\s+bin))\b/i;
+
/** An intent that says the typed text is a password, code or key (hosts that cannot see secure fields, X11). */
const SECRET_INTENT = /\b(pass(word|code|phrase)s?|pins?|one[-\s]?time\s+(code|password)|otp|2fa|mfa|verification\s+code|security\s+code|secret|api[-\s]?key|access\s+key|token|credentials?|cvv|cvc|card\s+number)\b/i;
@@ -314,49 +331,53 @@ export function looksLikeCredential(value) {
return classes >= 3 && entropy(word) >= 3.8 && !/^https?:\/\//i.test(word) && !/^[\w.+-]+@[\w-]+\.[\w.]+$/.test(word);
}
-/** A window title without unsaved-change markers and the app name suffix, for "same document" checks. */
-export function documentOf(title, appName) {
- let t = String(title ?? "").trim().replace(/^[*•●◦]\s*/, "").replace(/\s*[*•●◦]$/, "").replace(/\s+\(modified\)$/i, "");
- const app = String(appName ?? "").trim();
- if (app) for (const sep of [" - ", " — ", " – ", " | "]) if (t.toLowerCase().endsWith((sep + app).toLowerCase())) t = t.slice(0, -(sep + app).length);
- return t.replace(/\s+/g, " ").trim().toLowerCase();
-}
-
/**
* Whether one computer_act asks the person first, by permission mode:
* - careful: every act asks;
- * - normal: only paying, buying or ordering (the intent, or the text at a
- * click or Enter target) and typing a password (a secure field, text that
+ * - normal: only paying, buying or ordering (the intent, the text at a click
+ * or Enter/Space target, or, where the host reads no text there, a page
+ * title that says so) and typing a password (a secure field, text that
* looks like a credential, or an intent that says so); plus an act outside
* the approved window, which a session never does (acts go to the approved
- * window, other apps need their own open consent);
+ * window, other apps need their own open consent). With ASK_SEND_DELETE
+ * (or options.sendDelete) also OS-read send/permanent-delete targets and
+ * the Shift+Delete / Cmd+Backspace chords;
* - auto: never.
* In every mode the open-session consent, the deny list, hit-testing, the
* indicator and Stop still apply.
* `input`: { mode, action, intent, target: { text, secure }, window: { id,
- * app_id }, approved: { window_id, app_id } } (target.text undefined: the
- * host reads no text there).
+ * app_id, title }, approved: { window_id, app_id } } (target.text undefined:
+ * the host reads no text there). `options`: { sendDelete }.
* Returns { ask, reasons: [{ code, detail? }] }.
*/
-export function actApproval(input = {}) {
+export function actApproval(input = {}, options = {}) {
const mode = PERMISSION_MODES.includes(input.mode) ? input.mode : "normal";
if (mode === "auto") return { ask: false, reasons: [] };
if (mode === "careful") return { ask: true, reasons: [{ code: "careful" }] };
+ const sendDelete = options.sendDelete ?? ASK_SEND_DELETE;
const reasons = [];
const action = isRecord(input.action) ? input.action : {};
const target = isRecord(input.target) ? input.target : {};
+ const window = isRecord(input.window) ? input.window : null;
+ const key = action.type === "key" ? String(action.key ?? "").toLowerCase() : "";
+ const mods = new Set(Array.isArray(action.modifiers) ? action.modifiers : []);
const intentWords = riskyWords(input.intent);
if (intentWords.length) reasons.push({ code: "risky_intent", detail: intentWords.join(", ") });
- if (["click", "double_click", "right_click", "drag"].includes(action.type) || (action.type === "key" && String(action.key).toLowerCase() === "enter")) {
+ // Clicks, and Enter or Space on the focused control (no modifiers).
+ const presses = ["click", "double_click", "right_click", "drag"].includes(action.type) || ((key === "enter" || key === "space") && mods.size === 0);
+ if (presses) {
const targetWords = riskyWords(target.text);
if (targetWords.length) reasons.push({ code: "risky_target", detail: targetWords.join(", ") });
+ // No text the host could read there (X11): the page title decides whether this may be a payment page.
+ else if (typeof target.text !== "string" && action.type !== "drag" && window && (riskyWords(window.title).length || PAYMENT_PAGE.test(String(window.title ?? "")))) reasons.push({ code: "risky_page", detail: String(window.title ?? "").slice(0, 120) });
+ if (sendDelete && typeof target.text === "string" && (SEND_TARGET.test(target.text) || DELETE_TARGET.test(target.text))) reasons.push({ code: "send_delete_target", detail: target.text.slice(0, 80) });
}
+ if (sendDelete && action.type === "key" && ((mods.has("shift") && key === "delete") || (mods.has("meta") && (key === "backspace" || key === "delete")))) reasons.push({ code: "delete_chord" });
if (action.type === "type") {
if (target.secure === true) reasons.push({ code: "secure_field" });
if (looksLikeCredential(action.text)) reasons.push({ code: "credential_text" });
else if (target.secure !== true && SECRET_INTENT.test(String(input.intent ?? ""))) reasons.push({ code: "credential_intent" });
}
- const window = isRecord(input.window) ? input.window : null;
const approved = isRecord(input.approved) ? input.approved : null;
if (window && approved && ((approved.app_id && window.app_id && approved.app_id !== window.app_id) || (approved.window_id !== undefined && window.id !== undefined && String(approved.window_id) !== String(window.id)))) reasons.push({ code: "other_window" });
return { ask: reasons.length > 0, reasons };
@@ -367,6 +388,9 @@ export const ASK_REASONS = Object.freeze({
careful: "Careful mode asks before every action.",
risky_intent: "The agent says it will pay, buy or order something",
risky_target: "The target looks like a button that pays, buys or orders",
+ risky_page: "The page looks like a checkout or payment page",
+ send_delete_target: "The target looks like a button that sends, posts or deletes for good",
+ delete_chord: "The shortcut deletes for good.",
secure_field: "It would type into a password field.",
credential_text: "The text looks like a password or key.",
credential_intent: "The agent says it is typing a password, code or key.",
diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjs
index ad2a23ba..2d796654 100644
--- a/apps/server/src/computer-use/shared/portable-host.mjs
+++ b/apps/server/src/computer-use/shared/portable-host.mjs
@@ -311,21 +311,8 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
return mcpResult(observeResult({ session_id: s.id, observation: obs.ref, window: windowRef(w), screen: s.screen, scale: w.bounds.width / obs.model.width, dpi: w.dpi, monitor_id: w.monitor, cursor: await native.call("cursor").catch(() => null), os, expires_in_ms: l.observation_ttl_seconds * 1000, actions_left: Math.max(0, l.actions_per_session - s.actions), seconds_left: Math.max(0, Math.ceil((s.expiresAt - Date.now()) / 1000)) }), includeImage ? { data: obs.bytes, mime: obs.mime } : null);
}
- /** The on-screen text at (or within 24 px of) the action's point, and whether it is in a secure field. Unknown text: undefined. */
- /** Whether the control the last click focused (this window) is a secure field: { secure: true } or {}. */
- async function focusedTarget(observed, current, s) {
- const f = s?.focusPoint;
- if (!f || f.window !== String(current.id)) return {};
- const mx = ((f.screen.x - current.bounds.x) * observed.model.width) / current.bounds.width, my = ((f.screen.y - current.bounds.y) * observed.model.height) / current.bounds.height;
- if ((observed.secureFields ?? []).some((r) => mx >= r.x && mx <= r.x + r.width && my >= r.y && my <= r.y + r.height)) return { secure: true };
- const answer = await native.call("target", { points: [f.screen] }).catch(() => null);
- return Array.isArray(answer?.targets) && answer.targets[0]?.protected ? { secure: true } : {};
- }
-
- async function targetAt(observed, action, current, s) {
- const p = action.type === "drag" ? action.path.at(-1) : action.x !== undefined ? { x: action.x, y: action.y } : null;
- // Typing has no point: it goes to the control the last click in this window focused.
- if (!p) return action.type === "type" ? focusedTarget(observed, current, s) : {};
+ /** The on-screen text at (or within 24 px of) a point of the image, and whether it is in a secure field. Unknown text: undefined. */
+ async function targetAtPoint(observed, current, p) {
const inside = (r, pad) => p.x >= r.x - pad && p.x <= r.x + r.width + pad && p.y >= r.y - pad && p.y <= r.y + r.height + pad;
let secure = (observed.secureFields ?? []).some((r) => inside(r, 0));
if (!observed.texts) {
@@ -340,6 +327,21 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
return { text: near.map((t) => t.text).join(" "), ...(secure ? { secure } : {}) };
}
+ async function targetAt(observed, action, current, s) {
+ const p = action.type === "drag" ? action.path.at(-1) : action.x !== undefined ? { x: action.x, y: action.y } : null;
+ if (p) return targetAtPoint(observed, current, p);
+ // Typing, Enter and Space have no point: they go to the control the last click in this window focused
+ // (none after a Tab moved the focus: unknown).
+ const key = action.type === "key" ? String(action.key ?? "").toLowerCase() : "";
+ if (action.type !== "type" && key !== "enter" && key !== "space") return {};
+ const f = s?.focusPoint;
+ if (!f || f.window !== String(current.id)) return {};
+ const model = { x: ((f.screen.x - current.bounds.x) * observed.model.width) / current.bounds.width, y: ((f.screen.y - current.bounds.y) * observed.model.height) / current.bounds.height };
+ const found = await targetAtPoint(observed, current, model);
+ // Typing: only whether the field is secure (its label is not what is typed).
+ return action.type === "type" ? (found.secure ? { secure: true } : {}) : found;
+ }
+
async function actTool(s, args) {
const l = limits();
const started = Date.now();
@@ -382,7 +384,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
if (compare.stale) return finish({ ...common, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });
}
// Per-action approval by permission mode (host channel, never the engine's).
- const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: await targetAt(observed, action, current, s), window: { id: String(current.id), app_id: current.appId }, approved: s.approved });
+ const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: await targetAt(observed, action, current, s), window: { id: String(current.id), app_id: current.appId, title: current.title }, approved: s.approved });
let approval = { asked: false };
if (decision.ask) {
// The same action with the same intent, allowed a moment ago (a retry after a fresh observation), is not asked again.
@@ -424,6 +426,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
s.lastOperation = Date.now();
// Typing goes to the control the last click focused: its secure-field check reads this point.
if (["click", "double_click"].includes(action.type) && mapped.length === 1) s.focusPoint = { window: String(current.id), screen: mapped[0].screen };
+ else if (action.type === "key" && String(action.key).toLowerCase() === "tab") s.focusPoint = null;
const cursorAfter = await native.call("cursor").catch(() => null);
const done = { ...common, approval, point, path, cursor_before: cursorBefore, cursor_after: cursorAfter, ok: true, may_have_acted: action.type !== "wait" };
if (!args.observe_after) return finish({ ...done, next: "computer_observe" });
@@ -491,8 +494,10 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
update();
const grantKey = `${s.chat ?? ""}\0${preferred.appId}\0${s.mode}`;
const grant = s.chat ? grants.get(grantKey) : null;
+ // The server's bound now (it may have been lowered since): 0 = until the chat ends.
+ const fresh = grant && (l.chat_approval_seconds === 0 || Date.now() - grant.at < l.chat_approval_seconds * 1000);
let answer;
- if (grant && grant.until > Date.now()) answer = { decision: "chat", window_id: String(preferred.id), reused: true };
+ if (fresh) answer = { decision: "chat", window_id: String(preferred.id), reused: true };
else {
const pendingAnswer = new Promise((resolve) => { s.pendingApproval = { resolve }; });
const request = { session_id: s.id, request_id: randomUUID(), app: { app_id: preferred.appId, app_name: preferred.appName, bundle_or_exe: preferred.executable, pid: preferred.pid }, windows: windows.map((w) => ({ id: String(w.id), title: w.title })), window_id: String(preferred.id), mode: s.mode, purpose: s.purpose, os, seconds: l.session_seconds, chat_seconds: l.chat_approval_seconds };
@@ -525,9 +530,10 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
}
s.approval = answer.decision;
// "Allow for this chat": this app, this chat, until the chat ends (chat_approval_seconds 0), or for at most the
- // server's chat_approval_seconds. Kept in memory only, keyed to the chat: never another chat, never a restart.
- // A call no chat can be named for gets no grant (it works as "Allow once").
- if (answer.decision === "chat" && !answer.reused && s.chat) grants.set(grantKey, { until: l.chat_approval_seconds > 0 ? Date.now() + l.chat_approval_seconds * 1000 : Infinity });
+ // server's chat_approval_seconds (checked again at each reuse). Kept in memory only, keyed to the chat: never
+ // another chat, never a restart; the person's Stop and the server's switch clear it. A call no chat can be
+ // named for gets no grant (it works as "Allow once").
+ if (answer.decision === "chat" && !answer.reused && s.chat) grants.set(grantKey, { at: Date.now() });
s.phase = "working"; s.lastOperation = Date.now();
s.expiresAt = Date.now() + l.session_seconds * 1000;
s.approved = { window_id: String(s.window.id), app_id: s.window.appId };
@@ -637,7 +643,7 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
async function control({ session_id, action }) {
const s = sessions.get(session_id);
if (!s) throw new Error("This computer use session has ended.");
- if (action === "stop") return stop(s, "stopped", "person");
+ if (action === "stop") { grants.clear(); return stop(s, "stopped", "person"); }
if (action === "takeover") return pause(s, "person", "You took over. Choose Continue when you are ready.");
if (action === "hide" || action === "show") { s.hidden = action === "hide"; update(); return undefined; }
if (action !== "continue" || s.phase !== "paused") throw new Error("This action is unavailable.");
@@ -656,6 +662,8 @@ export async function createPortableHost({ native: firstNative, nativeFactory =
/** Ends every session (server switch off, app quitting). */
async function stopAll(by = "server", event = "stopped") {
+ // Stop by the person or the server: "Allow for this chat" is asked again.
+ if (by === "person" || by === "server") grants.clear();
for (const s of [...sessions.values()]) await stop(s, event, by);
}