A desktop coding agent with free access to frontier models.
TypeScript
fix: enter in forms and password fields
ogsamrat9d92e358d1ac
diff --git a/apps/desktop/electron/browser-task.mjs b/apps/desktop/electron/browser-task.mjsindex 813fd9e6..dd84421b 100644--- a/apps/desktop/electron/browser-task.mjs+++ b/apps/desktop/electron/browser-task.mjs@@ -58,8 +58,16 @@ export function observePage(id) {observation.observer = new MutationObserver(() => { observation.changed = true; });observation.observer.observe(document, { subtree: true, childList: true, attributes: true, characterData: true });globalThis[key] = observation;- const focused = document.activeElement instanceof HTMLElement && document.activeElement !== document.body ? controlName(document.activeElement) : "";- return { title: document.title.slice(0, 300), focusedName: focused, text: (document.body?.innerText || "").slice(0, 16000), elements,+ // Enter in a text field submits its form: the field's own label says nothing+ // about what that does, so a text field reports no name (the page title and+ // the form's submit button decide instead).+ const active = document.activeElement instanceof HTMLElement && document.activeElement !== document.body ? document.activeElement : null;+ const textEntry = active && (active instanceof HTMLTextAreaElement || active.isContentEditable || (active instanceof HTMLInputElement && !["submit", "button", "reset", "image", "checkbox", "radio", "file", "range", "color"].includes(active.type)));+ const submit = textEntry ? active.form?.querySelector('button[type="submit"],button:not([type]),input[type="submit"],input[type="image"]') : null;+ const focusedName = active && !textEntry ? controlName(active) : "";+ const focusedSubmitName = submit instanceof HTMLElement ? controlName(submit) : "";+ const focusedSensitive = active?.matches('input[type="password"],input[autocomplete="one-time-code"]') === true;+ return { title: document.title.slice(0, 300), focusedName, focusedSubmitName, focusedSensitive, text: (document.body?.innerText || "").slice(0, 16000), elements,viewport: { width: innerWidth, height: innerHeight }, hasPasswordField: [...document.querySelectorAll('input[type="password"],input[autocomplete="one-time-code"]')].some((element) => { const rect = element.getBoundingClientRect(); return rect.width > 0 && rect.height > 0; }),webmcp: "modelContext" in document ? "available" : "check_site_tools",limitations: ["DOM references cover the top document and open controls; use a fresh image for frames or canvas."] };@@ -122,7 +130,10 @@ const BROWSER_KEYS = { Enter: "enter", Space: "space", Tab: "tab", Escape: "escaexport function browserActInput(mode, action, observed) {// Keys go to the focused control and scrolls press nothing: a ref on them is ignored.const element = action.ref && (action.type === "click" || action.type === "fill") ? observed.elements.find((item) => item.ref === action.ref) : null;- const focused = action.type === "key" && (action.key === "Enter" || action.key === "Space") && observed.focusedName ? { text: observed.focusedName } : {};+ const presses = action.type === "key" && (action.key === "Enter" || action.key === "Space");+ const focused = presses && observed.focusedName ? { text: observed.focusedName } : {};+ // Enter from a text field submits its form: its submit button's name joins the page title.+ const submitName = action.type === "key" && action.key === "Enter" && !observed.focusedName ? observed.focusedSubmitName ?? "" : "";const act = action.type === "fill" ? { type: "type", text: action.text }: action.type === "key" ? { type: "key", key: BROWSER_KEYS[action.key] ?? String(action.key ?? "").toLowerCase() }: { type: action.type };@@ -131,7 +142,7 @@ export function browserActInput(mode, action, observed) {action: act,// No readable name (icon-only control): no text, so the page title decides, as on hosts that read none.target: element ? { text: element.name || undefined, secure: element.sensitive === true } : focused,- window: { title: observed.title ?? "" },+ window: { title: [observed.title ?? "", submitName].filter(Boolean).join(" - ") },};}@@ -310,7 +321,7 @@ export function createBrowserTaskHost({ getTab, tabsFor, ownerOf, activeFor, isVif (!await allowed(url)) fail("website_blocked", "Your organization does not allow this website.");if (revision !== tab.webMcpRevision || url !== tab.view.webContents.getURL()) fail("page_changed", "The page changed during observation. Observe again.");signal.throwIfAborted();- state.observation = { id, at: Date.now(), revision, url, title: page.title, focusedName: page.focusedName, viewport: page.viewport, elements: page.elements, digest };+ state.observation = { id, at: Date.now(), revision, url, title: page.title, focusedName: page.focusedName, focusedSubmitName: page.focusedSubmitName, focusedSensitive: page.focusedSensitive === true, viewport: page.viewport, elements: page.elements, digest };return { ok: true, tabId: tab.tabId, observationId: id, url: safeUrl(url), trust: TRUST, ...page, ...(image ? { image } : {}) };}/**@@ -414,13 +425,16 @@ export function createBrowserTaskHost({ getTab, tabsFor, ownerOf, activeFor, isVconst keys = { Enter: "Enter", Tab: "Tab", Escape: "Escape", ArrowDown: "Down", ArrowUp: "Up", ArrowLeft: "Left", ArrowRight: "Right", Backspace: "Backspace", Space: "Space" };if (action.type === "key" && !Object.hasOwn(keys, action.key)) fail("invalid_key", "Use Enter, Tab, Escape, arrows, Backspace or Space. System shortcuts are unavailable.");if (action.type === "scroll" && (!Number.isFinite(action.deltaY) || Math.abs(action.deltaY) > 1200)) fail("invalid_scroll", "Scroll distance must be between -1200 and 1200.");+ // Password and one-time-code fields are refused outright, before any ask.+ const targeted = action.ref && (action.type === "click" || action.type === "fill") ? observed.elements.find((item) => item.ref === action.ref) : null;+ if (targeted?.sensitive || (action.type === "key" && observed.focusedSensitive)) fail("sign_in_required", "Take over and sign in directly in the browser, then resume.");const decision = await actApproval(browserActInput(await modeNow(), action, observed));controller.signal.throwIfAborted();if (decision.ask) {if (!isVisible(tab.tabId)) fail("needs_attention", "This action needs your approval. Select this tab in its conversation's browser panel, then retry from a fresh observation.");publish(tab.tabId, "needs_attention", `Approve ${action.type}`);const why = (decision.reasons ?? []).map((reason) => reason.text).filter(Boolean).join(" ");- const accepted = await confirm({ tabId: tab.tabId, title: "Allow browser action?", message: `Allow ${action.type} on ${new URL(observed.url).origin}?`, detail: `${why ? `${why} ` : ""}Target: ${observed.elements.find((item) => item.ref === action.ref)?.name || (action.key ? `key ${action.key}` : `position ${action.x}, ${action.y}`)}.${action.type === "fill" ? ` Text to enter: ${action.text}` : ""} This can submit information or change website data. Review your requested task before allowing it.`, signal: controller.signal });+ const accepted = await confirm({ tabId: tab.tabId, title: "Allow browser action?", message: `Allow ${action.type} on ${new URL(observed.url).origin}?`, detail: `${why ? `${why} ` : ""}Target: ${observed.elements.find((item) => item.ref === action.ref)?.name || (action.key ? `key ${action.key}${observed.focusedName ? ` on ${observed.focusedName}` : ""}` : `position ${action.x}, ${action.y}`)}.${action.type === "fill" ? ` Text to enter: ${action.text}` : ""} This can submit information or change website data. Review your requested task before allowing it.`, signal: controller.signal });controller.signal.throwIfAborted();if (!accepted) fail("user_denied", "The browser action was not allowed.");}diff --git a/apps/desktop/electron/browser-task.test.mjs b/apps/desktop/electron/browser-task.test.mjsindex 5945e455..be4793ce 100644--- a/apps/desktop/electron/browser-task.test.mjs+++ b/apps/desktop/electron/browser-task.test.mjs@@ -6,7 +6,7 @@ import { actApproval, ASK_REASONS } from "../../server/src/computer-use/shared/pconst PAGE_URL = "https://shop.example.com/cart";-function fakeTab(elements, { title = "Example shop" } = {}) {+function fakeTab(elements, { title = "Example shop", page = {} } = {}) {let url = PAGE_URL;const calls = { inserted: [], keys: [], clicks: 0 };const webContents = {@@ -20,7 +20,7 @@ function fakeTab(elements, { title = "Example shop" } = {}) {debugger: { isAttached: () => false, attach() {}, detach() {}, sendCommand: async () => { calls.clicks += 1; } },executeJavaScriptInIsolatedWorld: async (_world, [{ code }]) => {if (code.startsWith("(function observePage")) {- return { title, text: "", elements, viewport: { width: 800, height: 600 }, hasPasswordField: false };+ return { title, text: "", elements, viewport: { width: 800, height: 600 }, hasPasswordField: false, ...page };}return { x: 10, y: 10 };},@@ -28,8 +28,8 @@ function fakeTab(elements, { title = "Example shop" } = {}) {return { tab: { tabId: "t1", view: { webContents }, webMcpRevision: 0 }, calls, setUrl: (next) => { url = next; } };}-function hostFor({ mode = "normal", visible = false, rules = true, allowedUrl = () => true, elements = [], title } = {}) {- const { tab, calls, setUrl } = fakeTab(elements, { title });+function hostFor({ mode = "normal", visible = false, rules = true, allowedUrl = () => true, elements = [], title, page } = {}) {+ const { tab, calls, setUrl } = fakeTab(elements, { title, page });const asked = [];const navigated = [];const host = createBrowserTaskHost({@@ -258,3 +258,30 @@ test("observed names include button values, alt text and tooltips, never typed frestore?.();}});++test("Normal: Enter in a labelled text field on a checkout page asks", async () => {+ const { act, asked } = hostFor({ mode: "normal", visible: true, elements: BUTTONS, title: "Checkout - Example shop", page: { focusedName: "", focusedSubmitName: "" } });+ const result = await act({ type: "key", key: "Enter" });+ assert.equal(result.ok, true, JSON.stringify(result));+ assert.equal(asked.length, 1);+});++test("Normal: Enter in a text field whose form submit says Place order asks, even off a checkout title", () => {+ const observed = { title: "Example shop", focusedName: "", focusedSubmitName: "Place order", elements: BUTTONS };+ assert.equal(actApproval(browserActInput("normal", { type: "key", key: "Enter" }, observed)).ask, true);+ assert.equal(actApproval(browserActInput("normal", { type: "key", key: "Enter" }, { ...observed, focusedSubmitName: "Search" })).ask, false);+});++test("password fields are refused before any ask", async () => {+ const secret = [{ ref: "e1", tag: "input", role: null, name: "Password", disabled: false, sensitive: true, bounds: { x: 0, y: 0, width: 20, height: 20 } }];+ for (const mode of ["careful", "normal"]) {+ const { act, asked } = hostFor({ mode, visible: true, elements: secret });+ const result = await act({ type: "fill", ref: "e1", text: "x" });+ assert.equal(result.code, "sign_in_required", mode);+ assert.equal(asked.filter((item) => item.title === "Allow browser action?").length, 0, mode);+ }+ const focused = hostFor({ mode: "careful", visible: true, elements: BUTTONS, page: { focusedSensitive: true } });+ const key = await focused.act({ type: "key", key: "Enter" });+ assert.equal(key.code, "sign_in_required");+ assert.equal(focused.asked.filter((item) => item.title === "Allow browser action?").length, 0);+});