Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Support reusable agent browser profiles (#132)

alexshpunt-clanker[bot]731c4209a06a
diff --git a/src/agent-profile-lock.ts b/src/agent-profile-lock.ts
new file mode 100644
index 0000000..708c98b
--- /dev/null
+++ b/src/agent-profile-lock.ts
@@ -0,0 +1,31 @@
+import { spawn, execFile, type ChildProcess } from "node:child_process";
+import { createHash } from "node:crypto";
+import { promisify } from "node:util";
+
+/** Serialize our launches across processes and refuse a profile already used by a running browser. */
+export async function lockAgentProfile(browser: "chrome" | "msedge" | "firefox", directory: string): Promise<ChildProcess> {
+ const name = "Global\\PiComputerUseBrowser-" + createHash("sha256").update(directory.toLowerCase()).digest("hex");
+ const script = `$m = [System.Threading.Mutex]::new($false, '${name}'); try { $acquired = $m.WaitOne(0) } catch [System.Threading.AbandonedMutexException] { $acquired = $true }; if (-not $acquired) { [Console]::WriteLine('busy'); exit 2 }; [Console]::WriteLine('locked'); [Console]::ReadLine() | Out-Null; $m.ReleaseMutex(); $m.Dispose()`;
+ const guard = spawn("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", script], { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
+ try {
+ await new Promise<void>((resolve, reject) => {
+ let output = "";
+ const timer = setTimeout(() => finish(new Error("Could not acquire the agent profile lock.")), 15_000);
+ const finish = (error?: Error) => { clearTimeout(timer); guard.stdout?.off("data", data); guard.off("error", failed); guard.off("exit", exited); error ? reject(error) : resolve(); };
+ const data = (chunk: Buffer) => { output += chunk.toString(); if (output.includes("locked")) finish(); else if (output.includes("busy")) finish(new Error("The agent browser profile is already in use.")); };
+ const failed = (error: Error) => finish(error);
+ const exited = () => finish(new Error("Could not acquire the agent profile lock."));
+ guard.stdout?.on("data", data); guard.once("error", failed); guard.once("exit", exited);
+ });
+ const escaped = directory.replace(/'/g, "''");
+ const processName = browser + ".exe";
+ const check = `$p = @(Get-CimInstance Win32_Process -ErrorAction Stop -Filter "Name='${processName}'"); if ($p | Where-Object { $null -eq $_.CommandLine }) { throw 'Cannot inspect browser process command lines' }; ConvertTo-Json -InputObject @($p | Where-Object { $_.CommandLine.IndexOf('${escaped}', [StringComparison]::OrdinalIgnoreCase) -ge 0 } | Select-Object -ExpandProperty ProcessId) -Compress`;
+ const { stdout } = await promisify(execFile)("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", check], { windowsHide: true, timeout: 15_000 });
+ const running = JSON.parse(stdout.trim() || "[]") as number[];
+ if (running.length) throw new Error("The agent browser profile is already in use; the running browser was not changed.");
+ return guard;
+ } catch (error) {
+ guard.stdin?.end("\n");
+ throw error;
+ }
+}
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index b2d2266..9e78bf6 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -343,6 +343,10 @@ export class ComputerResources {
private readonly viewports = new Map<string, Viewport>();
/** This machine's session once started; reference checks need it synchronously. */
private local?: ComputerSession;
+ /** A released session left a YOLO profile whose process/port could not be confirmed closed. */
+ private unconfirmedAgentCleanup = false;
+ /** Sessions being released remain visible to the Safe-mode guard until cleanup settles. */
+ private readonly closingSessions = new Set<ComputerSession>();
/** Give an installed app of one machine an opaque reference; paths themselves are not session references. */
private appRef(machine: string, id: string): string {
@@ -354,6 +358,12 @@ export class ComputerResources {
private readonly closeSession: () => Promise<ComputerResult>, private readonly openSandbox?: SandboxFactory,
private readonly startMachine?: MachineFactory, private readonly getMode: () => ComputerMode = () => "yolo") {}
+ /** Whether this run has an agent browser with a YOLO debugging port, including one still launching. */
+ hasUnapprovedAgentBrowser(): boolean {
+ return this.unconfirmedAgentCleanup || !!this.local?.hasUnapprovedAgentBrowser() ||
+ [...this.closingSessions].some(session => session.hasUnapprovedAgentBrowser()) ||
+ [...this.sandboxes.values(), ...this.machines.values()].some(environment => environment.session.hasUnapprovedAgentBrowser());
+ }
/**
* The session that issued a reference, with its machine. Nothing falls back to this machine: a reference that no
* session of this computer use holds is refused before anything is sent.
@@ -518,12 +528,18 @@ export class ComputerResources {
for (const [id, entry] of this.snapshots)
if (entry.target.window_ref !== undefined ? !this.remoteWindow(entry.target.window_ref) : entry.target.machine === LOCAL_MACHINE) this.retire(id);
own = await this.closeSession().catch(error => failure(error, this.getMode()));
+ if (this.local?.hasUnapprovedAgentBrowser()) this.unconfirmedAgentCleanup = true;
} else {
const machine = this.machines.get(name);
if (!machine) return report(this.getMode(), [], `${computerSource("machine", name)} was not started.`);
+ this.closingSessions.add(machine.session);
this.machines.delete(name);
this.forget(machine.session, name);
- own = await machine.close().catch(error => failure(error, this.getMode()));
+ try { own = await machine.close().catch(error => failure(error, this.getMode())); }
+ finally {
+ if (machine.session.hasUnapprovedAgentBrowser()) this.unconfirmedAgentCleanup = true;
+ this.closingSessions.delete(machine.session);
+ }
}
for (const [ref, app] of this.apps) if (app.machine === name) this.apps.delete(ref);
return report(this.getMode(), [...parts, [computerSource("machine", name), own]]);
@@ -555,9 +571,12 @@ export class ComputerResources {
return sandbox;
}
private async closeSandbox(sandbox: OpenedSandbox): Promise<ComputerResult> {
+ this.closingSessions.add(sandbox.session);
this.sandboxes.delete(sandbox.ref);
this.forget(sandbox.session);
- await sandbox.close();
+ try { await sandbox.close(); }
+ catch (error) { if (sandbox.session.hasUnapprovedAgentBrowser()) this.unconfirmedAgentCleanup = true; throw error; }
+ finally { this.closingSessions.delete(sandbox.session); }
return { content: [{ type: "text", text: `Sandbox ${computerSource("sandbox", sandbox.ref)} destroyed. Its windows and observations are gone.` }],
details: { status: "closed", mode: this.getMode(), sandbox_ref: sandbox.ref } };
}
diff --git a/src/computer-tools.ts b/src/computer-tools.ts
index b2a2aa1..95e0426 100644
--- a/src/computer-tools.ts
+++ b/src/computer-tools.ts
@@ -36,7 +36,7 @@ const openParameters = Type.Object({
debugging: Type.Optional(Type.Boolean({ description: "Start the WebView2 app at path with its debugging port open, so its window can be attached." })),
browser: Type.Optional(Type.Union([Type.Literal("chrome"), Type.Literal("msedge"), Type.Literal("firefox")], { description: "Open a new visible browser driven by Playwright; its window takes DOM reads and page actions." })),
url: Type.Optional(Type.String({ description: "http(s) page the new browser opens." })),
- profile: Type.Optional(Type.Literal("agent", { description: "YOLO: use the agent's own persistent browser profile, which keeps its logins between sessions. Omit for a disposable profile." })),
+ profile: Type.Optional(Type.Literal("agent", { description: "Persistent agent profile; in Safe the user enables and approves Chromium debugging; in YOLO Chromium's local debugging port permits access by local processes. Firefox agent profiles require YOLO. Omit for a disposable profile." })),
sandbox: Type.Optional(Type.Object({
import: Type.Optional(Type.Array(Type.Union([Type.String(), Type.Object({ host: Type.String(), guest: Type.Optional(Type.String()), writable: Type.Optional(Type.Boolean()) }, { additionalProperties: false })]),
{ maxItems: 8, description: "Host folders shared into the sandbox, read-only unless writable." })),
diff --git a/src/index.ts b/src/index.ts
index f283c8b..266c607 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -27,7 +27,8 @@ export default async function computerUse(pi: ExtensionAPI): Promise<void> {
const visuals = new VisualSettings();
registerImageSettingsCommand(pi, images);
registerVisualSettingsCommand(pi, visuals);
- const getMode = registerComputerMode(pi, async () => { visuals.clear(); });
+ let canEnterSafe = () => true;
+ const getMode = registerComputerMode(pi, async () => { visuals.clear(); }, () => canEnterSafe());
registerSetupCommand(pi);
let session: ComputerSession | undefined;
let loading: Promise<ComputerSession> | undefined;
@@ -54,6 +55,7 @@ export default async function computerUse(pi: ExtensionAPI): Promise<void> {
session = undefined;
return result;
}, openSandbox, connectMachine, getMode);
+ canEnterSafe = () => !provider.resources.hasUnapprovedAgentBrowser();
// Every extension has loaded by session_start, so an IDE that accepts the provider has done so by now.
let registered = false;
pi.on("session_start", () => {
diff --git a/src/machine.ts b/src/machine.ts
index 34c7d9c..98fad0f 100644
--- a/src/machine.ts
+++ b/src/machine.ts
@@ -24,6 +24,8 @@ export async function machineConfig(cwd: string, name: string): Promise<NonNulla
/** The worker files whose version decides the remote browser's behavior, relative to its package copy. */
const WORKER_FILES: [string, string][] = [
["src/web-worker.ts", fileURLToPath(new URL("./web-worker.ts", import.meta.url))],
+ ["src/agent-profile-lock.ts", fileURLToPath(new URL("./agent-profile-lock.ts", import.meta.url))],
+ ["src/browser-command-line.ts", fileURLToPath(new URL("./browser-command-line.ts", import.meta.url))],
["src/aria-match.ts", fileURLToPath(new URL("./aria-match.ts", import.meta.url))],
["src/web-geometry.ts", fileURLToPath(new URL("./web-geometry.ts", import.meta.url))],
["node_modules/playwright-core/package.json", createRequire(import.meta.url).resolve("playwright-core/package.json")],
diff --git a/src/mode.ts b/src/mode.ts
index 1bfe04c..eb829d8 100644
--- a/src/mode.ts
+++ b/src/mode.ts
@@ -2,7 +2,7 @@ import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-a
import type { ComputerMode } from "./session.ts";
/** Register a user command, not a model tool. Restore user policy on reload; never start a native runtime here. */
-export function registerComputerMode(pi: ExtensionAPI, clearFrame: () => Promise<void> = async () => {}): () => ComputerMode {
+export function registerComputerMode(pi: ExtensionAPI, clearFrame: () => Promise<void> = async () => {}, canEnterSafe: () => boolean = () => true): () => ComputerMode {
let mode: ComputerMode = "yolo";
const show = (ctx: ExtensionContext) => {
if (ctx.hasUI) ctx.ui.setStatus("computer-use", `Computer: ${mode.toUpperCase()}`);
@@ -14,6 +14,10 @@ export function registerComputerMode(pi: ExtensionAPI, clearFrame: () => Promise
const data = entry.data as { mode?: unknown } | undefined;
if (data?.mode === "safe" || data?.mode === "yolo") mode = data.mode;
}
+ if (mode === "safe" && !canEnterSafe()) {
+ mode = "yolo";
+ ctx.ui.notify("Safe mode cannot be restored while a YOLO agent browser is open or starting. Close it first, then select /computer-mode safe.", "warning");
+ }
await clearFrame();
show(ctx);
};
@@ -28,6 +32,10 @@ export function registerComputerMode(pi: ExtensionAPI, clearFrame: () => Promise
ctx.ui.notify("Usage: /computer-mode safe|yolo. Without arguments, show the current mode.", "warning");
return;
}
+ if (selected === "safe" && selected !== mode && !canEnterSafe()) {
+ ctx.ui.notify("Wait for pending input to finish and close any YOLO agent browser before selecting Safe; its local debugging port stays accessible until it closes. If cleanup was unconfirmed, verify it is closed and restart Pi.", "warning");
+ return;
+ }
if ((selected === "safe" || selected === "yolo") && selected !== mode) {
// Persist only the user's choice, not window ownership or model-supplied data.
pi.appendEntry("computer-use-mode", { mode: selected });
diff --git a/src/session.ts b/src/session.ts
index 81935f0..83dfab9 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -27,7 +27,8 @@ import { cropDisplay, nativeRegion, validateDetail, type DetailContext, type Det
import type { StateDiff } from "./successor.ts";
import { combine, evaluateNative, validateWait, type ConditionResult, type WaitResult, type WaitSpec } from "./ui-conditions.ts";
import { computerSource } from "./computer-address.ts";
-import { WebError, type WebAction, type WebBrowser, type WebLauncher, type WebLaunchOptions, type WebPage, type WebSnapshotOptions, type WebTarget } from "./web-browser.ts";
+import { WebError, type WebAction, type WebBrowser, type WebLauncher, type WebLaunchOptions, type WebOpened, type WebPage, type WebSnapshotOptions, type WebTarget } from "./web-browser.ts";
+import { ATTACH_APPROVAL_MS } from "./browser-command-line.ts";
import { derivedPng } from "./web-capture.ts";
import type { Derive } from "./web-geometry.ts";
@@ -174,7 +175,7 @@ interface Observation { texts?: Map<string, string>; id: string; tokens: Set<str
interface WindowState { id: string; nativeId: string; kind: "restore" | "management" }
/** The unannotated pixels of a window's latest capture with pixels, kept as the baseline of its next pixel comparison. */
export interface WindowFrame { observation_id: string; png: Buffer; size: { maxLongEdge: number; scale: number } }
-interface WindowEntry { ref: string; target: WindowTarget; identity: string; owned: boolean; observation?: Observation;
+interface WindowEntry { ref: string; target: WindowTarget; identity: string; owned: boolean; pendingApproval?: boolean; attachingApproval?: boolean; observation?: Observation;
visualBounds?: ScreenRect;
/** Size of the window's latest image capture: points of typed input are in its pixels. Outlives the observation. */
seen?: { width: number; height: number };
@@ -269,6 +270,13 @@ export class ComputerSession {
* Popup windows of the same process are ordinary native windows; their pages are reached as tabs of this browser.
*/
private readonly browsers = new Map<string, WebBrowser>();
+ private readonly unapprovedAgentBrowsers = new Set<WebBrowser>();
+ private unapprovedAgentLaunches = 0;
+ /** Native input already dispatched under the current policy, including long timelines on dialog windows. */
+ private nativeInputsInFlight = 0;
+ private unconfirmedUnsafeWork = false;
+ /** Safe cannot be selected while a YOLO agent profile has an unauthenticated debugging port. */
+ hasUnapprovedAgentBrowser(): boolean { return this.unconfirmedUnsafeWork || this.unapprovedAgentLaunches > 0 || this.nativeInputsInFlight > 0 || this.unapprovedAgentBrowsers.size > 0; }
private get platform(): NativePlatform { return this.driver.platform ?? "windows"; }
/** Label sources from a sandbox guest so they never pass for host windows or displays. */
@@ -361,6 +369,9 @@ export class ComputerSession {
return this.getMode() === "yolo" || entry.owned;
}
+ private inputAllowed(entry: WindowEntry): boolean {
+ return this.allowed(entry) && !(entry.pendingApproval && this.getMode() === "safe");
+ }
private denied(ref: string): ComputerResult {
return this.result("refused", "Safe mode allows only windows opened by this session. Only the user can change mode with /computer-mode.",
undefined, { window_ref: ref });
@@ -475,42 +486,57 @@ export class ComputerSession {
}
/**
- * Launch one new visible browser through Playwright with a disposable profile, or in YOLO the agent's persistent
- * profile, then bind its native window. Closing the session closes this browser and deletes a disposable profile.
- * attachBrowser covers browsers that already run.
+ * Launch one visible browser with a disposable or persistent agent profile and bind its native window.
+ * Safe agent profiles start normally and attach after approval. Closing the session closes the owned browser.
+ * attachBrowser also covers browsers that already run.
*/
openBrowser(options: WebLaunchOptions = {}, signal?: AbortSignal): Promise<ComputerResult> {
return this.queue(async () => {
if (signal?.aborted || !this.launchWeb || this.driver.windowLifetime === false)
return this.result("refused", "Opening a browser needs Windows window lifetime and the Playwright browser backend; nothing was launched.");
- if (options.profile && this.getMode() !== "yolo")
- return this.result("refused", "The agent's persistent browser profile keeps logins, so Safe mode allows only disposable profiles. Only the user can select YOLO with /computer-mode.");
+ const approval = options.profile === "agent" && this.getMode() === "safe";
+ if (approval && options.browser === "firefox") return this.result("refused", "Safe agent profiles require a Chromium browser with approval-based remote debugging.");
+ const unapproved = options.profile === "agent" && !approval && options.browser !== "firefox";
+ if (unapproved) this.unapprovedAgentLaunches++;
let launched: Awaited<ReturnType<WebLauncher>> | undefined;
try {
- launched = await this.launchWeb(options, signal);
+ launched = await this.launchWeb(approval ? { ...options, approval: true } : options, signal);
+ if (unapproved) this.unapprovedAgentBrowsers.add(launched.browser);
+ if (unapproved && this.getMode() !== "yolo") throw new Error("Mode changed while the unapproved browser was starting.");
const { pid } = launched.opened;
// Chromium creates its native window asynchronously after Playwright sees the first page.
let windows: Record<string, unknown>[] = [];
- for (let attempt = 0; attempt < 20 && !windows.length; attempt++) {
+ for (let attempt = 0; attempt < 20 && windows.length !== 1; attempt++) {
if (attempt) await delay(250, undefined, { signal });
- windows = ((await this.state("list_windows", { pid }, signal)).windows as Record<string, unknown>[] ?? []).filter(window => window.pid === pid);
+ // A prompt can appear before the browser frame, even as the only window of this process.
+ const frames = ((await this.state("list_windows", { pid }, signal)).windows as Record<string, unknown>[] ?? [])
+ .filter(window => window.pid === pid && typeof window.title === "string" && / - (?:Microsoft\u200b? Edge|Google Chrome|Mozilla Firefox)$/i.test(window.title));
+ const titled = launched.opened.title && frames.filter(window => (window.title as string).startsWith(launched!.opened.title));
+ windows = titled && titled.length === 1 ? titled : frames;
}
- if (windows.length !== 1) throw Error(`Expected one native window for browser process ${pid}, found ${windows.length}.`);
+ if (windows.length !== 1) throw Error(`Expected one identifiable browser frame for process ${pid}, found ${windows.length}.`);
const target = windows[0] as unknown as WindowTarget;
this.browsers.set(`${target.pid}:${target.window_id}`, launched.browser);
const entry = await this.register(target, signal);
if (!entry) throw Error("The browser window lifetime could not be established.");
+ if (unapproved && this.getMode() !== "yolo") throw new Error("Mode changed while the unapproved browser was starting.");
entry.owned = true;
- return this.result("observed", `Opened a new visible browser with ${options.profile ? "the agent's persistent profile (logins made in it stay for later sessions)" : "a disposable profile"}. Read its window with dom for page structure, or with image/ui/full for native pixels and Windows UI.`,
+ if (approval) { entry.pendingApproval = true; entry.web = undefined; }
+ return this.result("observed", `Opened a new visible browser with ${options.profile ? "the agent's persistent profile (logins made in it stay for later sessions)" : "a disposable profile"}. ${approval ? "Sign in if needed, enable remote debugging in the browser, then attach this window and approve the connection." : options.profile && options.browser !== "firefox" ? "The local debugging port lets other local processes control this browser and access its sessions." : ""} Read its window with dom for page structure, or with image/ui/full for native pixels and Windows UI.`,
{ operation: "open_browser", ...launched.opened }, { ...this.addressed(entry), opened: true });
} catch (error) {
const reason = error instanceof Error ? error.message : String(error);
- if (!launched) return this.result("refused", `Browser launch failed: ${reason}`, { operation: "open_browser" });
+ if (!launched) {
+ const unknown = error instanceof WebError && error.transport;
+ if (unknown && unapproved) this.unconfirmedUnsafeWork = true;
+ return this.result(unknown ? "unknown" : "refused", `Browser launch failed: ${reason}`, { operation: "open_browser" });
+ }
for (const [key, browser] of this.browsers) if (browser === launched.browser) this.browsers.delete(key);
const cleanup = await launched.browser.close().then(() => "confirmed", () => "unconfirmed");
- return this.result(cleanup === "confirmed" ? "refused" : "unknown", `The browser started but its window could not be bound, so it was closed (cleanup ${cleanup}). ${reason}`,
+ if (cleanup === "confirmed") this.unapprovedAgentBrowsers.delete(launched.browser);
+ return this.result(cleanup === "confirmed" ? "refused" : "unknown", `The browser started but its window could not be bound (cleanup ${cleanup}). ${reason}`,
{ operation: "open_browser", cleanup });
- }
+ } finally { if (unapproved) this.unapprovedAgentLaunches--; }
});
}
/**
@@ -520,32 +546,59 @@ export class ComputerSession {
* Closing the session detaches; the browser and its tabs stay open.
*/
async attachBrowser(ref: string, signal?: AbortSignal): Promise<ComputerResult> {
- const prepared = await this.queue<ComputerResult | { pid: number; title: string }>(async () => {
+ const prepared = await this.queue<ComputerResult | { pid: number; title: string; owned?: WebBrowser }>(async () => {
if (signal?.aborted || !this.launchWeb || this.driver.windowLifetime === false)
return this.result("refused", "Attaching needs Windows window lifetime and the Playwright browser backend; nothing was attached.");
- if (this.getMode() !== "yolo")
- return this.result("refused", "Attaching Playwright to an existing browser exposes its signed-in sessions, so Safe mode allows only browsers the session opened. Only the user can select YOLO with /computer-mode.");
const entry = await this.current(ref, signal);
if (!entry) return this.result("refused", "Unknown or closed window. Discover again.");
if (entry.web) return this.result("refused", "Playwright already works in this browser; read the window with dom.", undefined, this.addressed(entry));
+ if (entry.owned && entry.pendingApproval) {
+ if (entry.attachingApproval) return this.result("refused", "An approval connection is already pending.");
+ const owned = this.browsers.get(`${entry.target.pid}:${entry.target.window_id}`);
+ if (!owned) return this.result("refused", "The owned browser is no longer available.");
+ entry.attachingApproval = true;
+ return { pid: entry.target.pid, title: "", owned };
+ }
+ if (this.getMode() !== "yolo") return this.result("refused", "Safe mode attaches only to its own agent profile after approval.");
const windows = (await this.state("list_windows", { pid: entry.target.pid }, signal)).windows as Record<string, unknown>[] ?? [];
const title = windows.find(window => window.window_id === entry.target.window_id)?.title;
return { pid: entry.target.pid, title: typeof title === "string" ? title : "" };
});
if ("content" in prepared) return prepared;
let launched: Awaited<ReturnType<WebLauncher>>;
- try { launched = await this.launchWeb!({ attach: prepared }, signal); }
- catch (error) { return this.result("refused", `Attaching failed: ${error instanceof Error ? error.message : String(error)}`, { operation: "attach_browser" }); }
+ try {
+ if (prepared.owned) {
+ const opened = await prepared.owned.request<WebOpened>("approve", {}, signal, ATTACH_APPROVAL_MS + 30_000);
+ launched = { browser: prepared.owned, opened };
+ } else launched = await this.launchWeb!({ attach: { pid: prepared.pid, title: prepared.title } }, signal);
+ }
+ catch (error) {
+ const unknown = error instanceof WebError && (error.transport || error.timeout);
+ let cleanup: string | undefined;
+ if (prepared.owned) await this.queue(async () => {
+ const entry = this.entries.get(ref);
+ if (entry) entry.attachingApproval = false;
+ if (unknown) {
+ if (entry) this.browsers.delete(`${entry.target.pid}:${entry.target.window_id}`);
+ cleanup = await prepared.owned!.close().then(() => "confirmed", () => "unconfirmed");
+ this.entries.delete(ref);
+ }
+ });
+ return this.result(unknown ? "unknown" : "refused", "Attaching failed: " + (error instanceof Error ? error.message : String(error))
+ + (cleanup ? ` Owned browser cleanup ${cleanup}.` : ""), { operation: "attach_browser", ...(cleanup ? { cleanup } : {}) });
+ }
return this.queue(async () => {
const entry = this.entries.get(ref);
- if (!entry || entry.web) {
+ if (!entry || entry.web || (!prepared.owned && this.getMode() !== "yolo")) {
const cleanup = await launched.browser.close().then(() => "confirmed", () => "unconfirmed");
- return this.result("refused", `The window closed or Playwright attached to it meanwhile, so this attachment was released (detach ${cleanup}).`, { operation: "attach_browser" });
+ return this.result(cleanup === "confirmed" ? "refused" : "unknown", `The window, attachment or access mode changed while attaching; the connection was released (cleanup ${cleanup}).`, { operation: "attach_browser" });
}
this.browsers.set(`${entry.target.pid}:${entry.target.window_id}`, launched.browser);
entry.web = launched.browser;
+ entry.pendingApproval = false;
+ entry.attachingApproval = false;
const via = { cdp: "its debugging endpoint", toggle: "the remote debugging turned on in the browser", extension: "the Playwright Extension", vscode: "VS Code's browser bridge" }[launched.opened.via ?? "extension"];
- return this.result("delivered", `Attached Playwright to this browser through ${via}. Read the window with dom for its active tab; every tab is listed there and switch_tab changes the active one. Closing the computer session detaches without closing the browser.`,
+ return this.result(launched.opened.navigation_error ? "unknown" : "delivered", `Attached Playwright to this browser through ${via}. ${launched.opened.navigation_error ? `The requested navigation outcome is unknown: ${launched.opened.navigation_error}. Do not replay it without checking the page. ` : ""}Read the window with dom for its active tab; every tab is listed there and switch_tab changes the active one. ${prepared.owned ? "Closing the computer session closes this owned browser." : "Closing the computer session detaches without closing the browser."}`,
{ operation: "attach_browser", ...launched.opened }, this.addressed(entry));
});
}
@@ -1369,7 +1422,7 @@ export class ComputerSession {
try { entry = await this.current(ref, signal); }
catch (error) { return this.result("refused", `Window lifetime check failed: ${error instanceof Error ? error.message : String(error)}`); }
if (!entry) return this.entries.has(ref) ? this.denied(ref) : this.result("refused", "Unknown or closed window. Discover again.");
- if (!entry.web) return this.result("refused", "DOM is available only in browsers opened or attached with computer_open (attaching is YOLO). Use image/ui/full Reads for other windows.",
+ if (!entry.web) return this.result("refused", "DOM is available in browsers opened or attached with computer_open. A Safe agent profile needs an approved connection first. Use image/ui/full Reads for other windows.",
undefined, this.addressed(entry));
return entry;
}
@@ -1745,7 +1798,8 @@ export class ComputerSession {
if (browser) {
this.browsers.delete(key);
entry.web = undefined; entry.dom = undefined;
- await browser.close().catch(() => undefined);
+ await browser.close();
+ this.unapprovedAgentBrowsers.delete(browser);
}
if (!this.host) {
if (browser) { this.entries.delete(ref); return this.result("delivered", "The browser this session opened was closed.", undefined, this.addressed(entry)); }
@@ -1993,6 +2047,7 @@ export class ComputerSession {
if (signal?.aborted) return this.result("refused", "Cancelled before action dispatch.");
const entry = this.entries.get(action.window_ref);
if (entry && !this.allowed(entry)) return this.denied(entry.ref);
+ if (entry?.pendingApproval && this.getMode() === "safe") return this.result("refused", "The user must enable and approve this browser connection; Safe mode does not send input to it before approval.");
const observed = entry?.observation;
if (!entry || !observed || action.observation_id !== observed.id)
return this.result("refused", "Stale or missing observation_id/window_ref. Observe again.");
@@ -2072,11 +2127,11 @@ export class ComputerSession {
let capture = observed.captureId;
for (const [index, step] of commands.entries()) {
if (signal?.aborted) return finish(this.result("refused", "Cancelled."), "Cancelled before the next dispatch.");
- if (!this.allowed(entry)) return finish(this.denied(entry.ref), "Permission revoked before the next dispatch.");
+ if (!this.inputAllowed(entry)) return finish(this.denied(entry.ref), "Permission revoked before the next dispatch.");
if (batch && performance.now() - started >= SEQUENCE_ELAPSED_MS) return finish(this.result("refused", "Sequence time budget reached."), `${SEQUENCE_ELAPSED_MS / 60000} min elapsed budget reached; no more steps dispatched.`);
if (step.wait_ms) await delay(step.wait_ms, undefined, { signal });
const current = await this.current(entry.ref, signal);
- if (!this.allowed(entry)) return finish(this.denied(entry.ref), "Permission revoked before dispatch.");
+ if (!this.inputAllowed(entry)) return finish(this.denied(entry.ref), "Permission revoked before dispatch.");
if (!current) return finish(this.result("refused", "Window closed or its native handle was reused. Discover again."), "Window lifetime changed.");
if (signal?.aborted) return finish(this.result("refused", "Cancelled."), "Cancelled before dispatch.");
// The native capture is single-use: the first step that binds to it spends it, later pixel steps use the window's current mapping.
@@ -2087,7 +2142,7 @@ export class ComputerSession {
const activityAction = presentationAction(step);
let highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, observed.bounds, signal, mark, undefined, activityAction);
if (signal?.aborted) return finish(this.result("refused", "Cancelled during action mark; input was not dispatched."), "Cancelled during action mark.");
- if (!this.allowed(entry)) return finish(this.denied(entry.ref), "Permission revoked during action mark.");
+ if (!this.inputAllowed(entry)) return finish(this.denied(entry.ref), "Permission revoked during action mark.");
if (batch && performance.now() - started >= SEQUENCE_ELAPSED_MS) return finish(this.result("refused", "Sequence time budget reached."), `${SEQUENCE_ELAPSED_MS / 60000} min elapsed budget reached during admission; no more steps dispatched.`);
// Input may change the page, so the window's DOM snapshot retires with its native capture.
entry.observation = undefined; entry.detail = undefined; entry.dom = undefined;
@@ -2103,9 +2158,21 @@ export class ComputerSession {
// A sequence stops between steps: its current step finishes and releases its own keys and buttons,
// and the driver connection survives the cancellation. A single action stays cancellable in flight.
const nativeSignal = batch ? undefined : signal;
- const native = await this.driver.call(name, args, nativeSignal);
+ const yoloDispatch = this.getMode() === "yolo";
+ this.nativeInputsInFlight++;
+ let native: DriverResult;
+ try {
+ native = await this.driver.call(name, args, nativeSignal);
+ const effect = String(native.structuredContent?.effect), status = String(native.structuredContent?.status);
+ if (yoloDispatch && ((native.isError && effect !== "refused" && status !== "refused") || ["unknown", "partial"].includes(status) || ["unknown", "partial"].includes(effect)))
+ this.unconfirmedUnsafeWork = true;
+ } catch (error) {
+ // Cancellation or transport loss may leave a scheduled native timeline running after this promise rejects.
+ if (yoloDispatch) this.unconfirmedUnsafeWork = true;
+ throw error;
+ } finally { this.nativeInputsInFlight--; }
row.completion_ms = performance.now() - started;
- if (!this.allowed(entry) || (!batch && signal?.aborted)) return finish(this.result("unknown",
+ if (!this.inputAllowed(entry) || (!batch && signal?.aborted)) return finish(this.result("unknown",
"Input was dispatched before permission was revoked or cancellation. Outcome withheld; native input may still have completed. Do not replay blindly."),
"In-flight outcome withheld after revocation/cancellation.");
const state = native.structuredContent ?? {};
@@ -2141,7 +2208,7 @@ export class ComputerSession {
}
highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, observed.bounds, signal,
{ kind: "action", ...(resolvedPoint ? { point: resolvedPoint } : step.element_token && mark.region ? { region: mark.region } : {}) }, undefined, activityAction, status);
- if (!this.allowed(entry) || (!batch && signal?.aborted)) {
+ if (!this.inputAllowed(entry) || (!batch && signal?.aborted)) {
entry.lastAction = { reason: "In-flight pointer outcome was withheld after revocation or cancellation." };
return finish(this.result("unknown", "Input was dispatched before revocation or cancellation. Do not replay blindly."), "In-flight outcome withheld after input.");
}
@@ -2251,6 +2318,7 @@ export class ComputerSession {
const browsers = [...this.browsers.values()];
this.browsers.clear();
const cleanup = await Promise.allSettled(browsers.map(browser => browser.close()));
+ browsers.forEach((browser, i) => { if (cleanup[i].status === "fulfilled") this.unapprovedAgentBrowsers.delete(browser); });
const modes = [...this.replacedModes];
this.replacedModes.clear();
const restored = await Promise.allSettled(modes.map(([device, mode]) => this.host!.displayModes.restore(device, mode)));
@@ -2261,7 +2329,7 @@ export class ComputerSession {
const unrestored = modes.filter((_, i) => restored[i].status === "rejected").map(([device, mode]) => `${device} (${mode.width}x${mode.height})`);
if (unrestored.length) return this.result("unknown", `Native runtime released, but restoring the display mode of ${unrestored.join(", ")} failed. Signing out of Windows also restores it.`);
if (!files) return this.result("unknown", "Native runtime released, but removing the session's file folder failed.");
- if (failed) return this.result("unknown", `Native runtime released, but cleanup of ${failed} of ${browsers.length} Playwright browsers is unconfirmed.`);
+ if (failed || this.hasUnapprovedAgentBrowser()) return this.result("unknown", `Native runtime released, but prior browser or input cleanup is unconfirmed (${failed} of ${browsers.length} managed browsers failed). A browser, local debugging port or scheduled input may still be active; verify before restarting this session.`);
return this.result("closed", "Owned computer-use runtime and session-opened browsers released; attached browsers were detached and stay open. Other application documents were not closed or saved.");
});
}
diff --git a/src/web-browser.ts b/src/web-browser.ts
index 91c345d..57c3030 100644
--- a/src/web-browser.ts
+++ b/src/web-browser.ts
@@ -146,11 +146,13 @@ export interface WebDownload {
/** Installed Chrome or Edge launched by Playwright, or Playwright's own Firefox build (setup part firefox). */
export type WebChannel = "chrome" | "msedge" | "firefox";
/**
- * Browser launch request. `profile: "agent"` uses the agent's own persistent profile on that machine (YOLO only):
+ * Browser launch request. `profile: "agent"` uses the agent's own persistent profile on that machine:
* logins made there stay for later sessions; it is never the user's own profile.
*/
export interface WebLaunchOptions {
browser?: WebChannel; url?: string; profile?: "agent";
+ /** Internal: launch the owned agent profile normally for an approval-based connection. */
+ approval?: boolean;
/**
* Attach to a running Chromium-based browser instead of launching one, embedded ones included: the window's process
* and title. See the worker's attach for the routes.
@@ -180,6 +182,8 @@ export interface WebOpened {
pid: number; version: string; url: string; title: string;
/** How Playwright reached an attached browser. */
via?: "cdp" | "toggle" | "extension" | "vscode";
+ /** Approval succeeded, but the requested navigation may or may not have reached the page. */
+ navigation_error?: string;
}
/** Starts one owned browser; the session binds its native window. */
export type WebLauncher = (options: WebLaunchOptions, signal?: AbortSignal) => Promise<{ browser: WebBrowser; opened: WebOpened }>;
@@ -412,7 +416,7 @@ export class WebBrowser {
this.pending.delete(response.id!);
// Playwright appends a colored call log; keep the plain first part for agents.
const message = String(response.error?.message ?? "Browser operation failed.").replace(/\x1b\[[0-9;]*m/g, "").split("\nCall log:")[0].trim();
- if (response.error) request.reject(new WebError(message, response.error.name === "TimeoutError"));
+ if (response.error) request.reject(new WebError(message, response.error.name === "TimeoutError", response.error.name === "ApprovalOutcomeUnknown"));
else request.resolve(response.value);
}
});
@@ -444,12 +448,13 @@ export class WebBrowser {
const browser = new WebBrowser(child, options.remote?.runner);
try {
const opened = await browser.request<WebOpened>("open", options.attach ? { attach: options.attach }
- : { browser: options.browser ?? "chrome", ...(options.url ? { url: options.url } : {}), ...(options.profile ? { profile: options.profile } : {}) },
+ : { browser: options.browser ?? "chrome", ...(options.url ? { url: options.url } : {}), ...(options.profile ? { profile: options.profile } : {}), ...(options.approval ? { approval: true } : {}) },
signal, options.attach ? ATTACH_APPROVAL_MS + 30_000 : 60_000);
if (!Number.isSafeInteger(opened.pid) || opened.pid <= 0) throw new WebError("Browser launch did not report a process id.");
return { browser, opened };
} catch (error) {
- await browser.close().catch(() => undefined);
+ try { await browser.close(); }
+ catch (cleanup) { throw new WebError(`Browser launch failed and cleanup is unconfirmed: ${cleanup instanceof Error ? cleanup.message : String(cleanup)}`, false, true); }
throw error;
}
}
@@ -485,7 +490,7 @@ export class WebBrowser {
if (!this.alive) return Promise.reject(new WebError("The browser worker is gone. Open a new browser."));
const id = ++this.serial;
return new Promise<T>((resolve, reject) => {
- const lost = (message: string) => { this.pending.delete(id); reject(new WebError(message, false, true)); };
+ const lost = (message: string) => { this.pending.delete(id); settle(); reject(new WebError(message, false, true)); };
const abort = () => lost("Cancelled after the browser received the request; it may still complete.");
const timer = setTimeout(() => lost("The browser worker did not answer in time; the request may still complete."), timeoutMs);
const settle = () => { clearTimeout(timer); signal?.removeEventListener("abort", abort); };
@@ -497,12 +502,13 @@ export class WebBrowser {
/** Close the browser and delete its profile, or detach from an attached one, then wait for the worker to exit. Idempotent. */
async close(): Promise<void> {
+ let failure: unknown;
if (this.alive) {
- await this.request("close", {}, undefined, 30_000).catch(() => undefined);
+ await this.request("close", {}, undefined, 30_000).catch(error => { failure = error; });
this.child.stdin.end();
}
const timer = setTimeout(() => this.child.kill(), 30_000);
try { await this.exited; } finally { clearTimeout(timer); }
- if (this.child.exitCode !== 0) throw new WebError(`Browser cleanup is unconfirmed. ${this.stderr.trim()}`.trim());
+ if (failure || this.child.exitCode !== 0) throw new WebError(`Browser cleanup is unconfirmed. ${failure instanceof Error ? failure.message : this.stderr.trim()}`.trim());
}
}
diff --git a/src/web-worker.ts b/src/web-worker.ts
index 190854f..4b1e8f2 100644
--- a/src/web-worker.ts
+++ b/src/web-worker.ts
@@ -1,7 +1,8 @@
// Windows Node worker: one visible Playwright browser with a disposable profile, the agent's persistent one, or a
// running Chrome/Edge it attached to. EOF closes an owned browser (deleting a disposable profile) or detaches.
import readline from "node:readline";
-import { execFile } from "node:child_process";
+import { execFile, spawn, type ChildProcess } from "node:child_process";
+import { createServer } from "node:net";
import { createRequire } from "node:module";
import { promisify } from "node:util";
import { mkdir, mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises";
@@ -9,6 +10,7 @@ import { tmpdir } from "node:os";
import path from "node:path";
import { chromium, firefox, type Browser, type BrowserContext, type Dialog, type Download, type FrameLocator, type Locator, type Page } from "playwright-core";
import { ATTACH_APPROVAL_MS, parseBrowserCommandLine } from "./browser-command-line.ts";
+import { lockAgentProfile } from "./agent-profile-lock.ts";
import type { WebAction, WebDownload, WebEvaluation, WebLaunchOptions, WebLogEntry, WebSnapshotOptions, WebTab, WebTarget, WebTimelineEvent } from "./web-browser.ts";
import type { ConditionResult, UiCondition } from "./ui-conditions.ts";
import { ariaTreeFromYaml, matchesAriaTemplate } from "./aria-match.ts";
@@ -21,6 +23,12 @@ let context: BrowserContext | undefined;
let profile: string | undefined;
/** A running browser this worker attached to; closing only disconnects from it. */
let attached: Browser | undefined;
+/** The agent-profile Chromium process launched by this worker, not an attached personal browser. */
+let ownedProcess: ChildProcess | undefined;
+let profileGuard: ChildProcess | undefined;
+let ownedVerified = false;
+let approvalDirectory: string | undefined;
+let approvalUrl: string | undefined;
/**
* Playwright's own extension relay (the Playwright Extension's connect flow). It is not part of the public API, so it
* is reached through the bundle's tools export rather than reimplemented here.
@@ -174,17 +182,134 @@ function locate(page: Page, target: WebTarget): Locator {
return parts.reduce((all, part) => all.and(part));
}
-async function open({ browser, url, profile: kind, attach: target }: WebLaunchOptions) {
+/** Reserve a nonzero loopback port; the browser must prove it owns the endpoint after the reservation is released. */
+async function freePort(): Promise<number> {
+ const server = createServer();
+ await new Promise<void>((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); });
+ const address = server.address();
+ if (!address || typeof address === "string") throw new Error("Could not reserve a browser debugging port.");
+ await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
+ return address.port;
+}
+
+function browserExecutable(browser: "chrome" | "msedge"): string {
+ const registry = (createRequire(import.meta.url)("playwright-core/lib/coreBundle") as {
+ registry: { registry: { findExecutable(name: string): { executablePathOrDie(language: string): string } | undefined } }
+ }).registry.registry;
+ const executable = registry.findExecutable(browser)?.executablePathOrDie("javascript");
+ if (!executable) throw new Error("Unsupported browser channel: " + browser);
+ return executable;
+}
+
+/** Start the owned agent profile without Playwright's pipe; only the spawned process may supply the endpoint. */
+async function agentChromium(browser: "chrome" | "msedge", directory: string): Promise<Browser> {
+ const executable = browserExecutable(browser);
+ const port = await freePort();
+ const child = spawn(executable, ["--user-data-dir=" + directory, "--remote-debugging-address=127.0.0.1",
+ "--remote-debugging-port=" + port, "--no-first-run", "--no-default-browser-check",
+ ...(browser === "msedge" ? ["--disable-sync"] : []), "--disable-back-forward-cache", "--window-size=1100,800", "about:blank"],
+ { stdio: ["ignore", "ignore", "pipe"], windowsHide: false });
+ ownedProcess = child;
+ const endpoint = await new Promise<string>((resolve, reject) => {
+ let log = "";
+ const timer = setTimeout(() => finish(new Error("The owned browser did not expose its debugging endpoint.")), 30_000);
+ const finish = (error?: Error, value?: string) => {
+ clearTimeout(timer); child.stderr?.off("data", data); child.off("error", failed); child.off("exit", exited);
+ error ? reject(error) : resolve(value!);
+ };
+ const data = (chunk: Buffer) => {
+ log = (log + chunk.toString()).slice(-16_000);
+ const match = log.match(/DevTools listening on (ws:\/\/127\.0\.0\.1:(\d+)\/devtools\/browser\/[a-zA-Z0-9-]+)/);
+ if (match && Number(match[2]) === port) finish(undefined, match[1]);
+ };
+ const failed = (error: Error) => finish(error);
+ const exited = () => finish(new Error("The browser exited before its debugging endpoint was ready; its profile may already be in use."));
+ child.stderr?.on("data", data); child.once("error", failed); child.once("exit", exited);
+ });
+ const connected = await chromium.connectOverCDP(endpoint);
+ attached = connected;
+ const cdp = await connected.newBrowserCDPSession();
+ try {
+ const { processInfo } = await cdp.send("SystemInfo.getProcessInfo");
+ if (processInfo.find(item => item.type === "browser")?.id !== child.pid)
+ throw new Error("The debugging endpoint does not belong to the browser this session launched.");
+ } finally { await cdp.detach(); }
+ ownedVerified = true;
+ return connected;
+}
+/** Launch normally. The user may sign in before enabling Chrome's approved debugging bridge. */
+async function approvalBrowser(browser: "chrome" | "msedge", directory: string, url?: string) {
+ const child = spawn(browserExecutable(browser), ["--user-data-dir=" + directory, "--no-first-run", "--no-default-browser-check",
+ browser === "chrome" ? "chrome://inspect/#remote-debugging" : "edge://inspect/#remote-debugging"],
+ { stdio: "ignore", windowsHide: false });
+ ownedProcess = child;
+ approvalDirectory = directory;
+ approvalUrl = url;
+ await new Promise<void>((resolve, reject) => { child.once("spawn", resolve); child.once("error", reject); });
+ if (!child.pid) throw new Error("The browser did not report its process id.");
+ return { pid: child.pid, version: "", url: "", title: "" };
+}
+
+/** Connect only to the approval bridge owned by this session's browser process. */
+async function approve() {
+ const child = ownedProcess, directory = approvalDirectory;
+ if (!child?.pid || !directory || child.exitCode !== null || child.signalCode !== null)
+ throw new Error("The owned agent browser is no longer running.");
+ const [portText, wsPath] = (await readFile(path.join(directory, "DevToolsActivePort"), "utf8").catch(() => "")).trim().split("\n");
+ const port = Number(portText);
+ if (!Number.isInteger(port) || port < 1 || port > 65535 || !/^\/devtools\/browser\/[a-zA-Z0-9-]+$/.test(wsPath ?? ""))
+ throw new Error("Enable remote debugging in the agent browser's inspect page before attaching.");
+ const tree = await processTree(child.pid);
+ if (!tree.ports.includes(port)) throw new Error("The approval endpoint is not owned by this browser; turn remote debugging off and on again.");
+ const connected = await chromium.connectOverCDP("ws://127.0.0.1:" + port + wsPath, { timeout: ATTACH_APPROVAL_MS }).catch(error => {
+ const failure = new Error(`Approval outcome is unknown: ${error instanceof Error ? error.message : String(error)}`);
+ failure.name = "ApprovalOutcomeUnknown";
+ throw failure;
+ });
+ // Approval has occurred. Retain the connection immediately so cleanup can always disconnect it.
+ attached = connected;
+ try {
+ const cdp = await connected.newBrowserCDPSession();
+ try {
+ const { processInfo } = await cdp.send("SystemInfo.getProcessInfo");
+ if (processInfo.find(item => item.type === "browser")?.id !== child.pid)
+ throw new Error("The approval endpoint does not belong to the owned browser.");
+ } finally { await cdp.detach().catch(() => undefined); }
+ const owned = connected.contexts()[0];
+ if (!owned) throw new Error("The approved browser exposed no tabs.");
+ context = owned; ownedVerified = true; approvalDirectory = undefined;
+ trackTabs(owned);
+ active = tabs[0] ?? adopt(await owned.newPage());
+ let navigation_error: string | undefined;
+ if (approvalUrl) {
+ const requested = approvalUrl; approvalUrl = undefined;
+ try { await active.page.goto(requested, { waitUntil: "domcontentloaded" }); }
+ catch (error) { navigation_error = (error instanceof Error ? error.message : String(error)).replace(/\x1b\[[0-9;]*m/g, "").split("\nCall log:")[0].trim(); }
+ }
+ return { pid: child.pid, version: connected.version(), url: active.page.url(), title: await active.page.title().catch(() => ""), via: "toggle", ...(navigation_error ? { navigation_error } : {}) };
+ } catch (error) {
+ const failure = new Error(`Approval may have succeeded before attachment failed: ${error instanceof Error ? error.message : String(error)}`);
+ failure.name = "ApprovalOutcomeUnknown";
+ throw failure;
+ }
+}
+async function open({ browser, url, profile: kind, approval, attach: target }: WebLaunchOptions) {
if (target) return attach(target);
if (context) throw new Error("This worker already owns a browser.");
// The agent's persistent profile lives in its own folder per browser and is kept on close; a disposable one is deleted.
const directory = kind === "agent"
? path.join(process.env.LOCALAPPDATA ?? tmpdir(), "PiComputerUse", "browser-profiles", browser ?? "chrome")
: profile = await mkdtemp(path.join(tmpdir(), "pi-browser-"));
- if (kind === "agent") await mkdir(directory, { recursive: true });
+ if (kind === "agent") {
+ await mkdir(directory, { recursive: true });
+ profileGuard = await lockAgentProfile(browser ?? "chrome", directory);
+ }
+ if (kind === "agent" && approval && browser !== "firefox") return approvalBrowser(browser ?? "chrome", directory, url);
// Playwright's default flags trade browser security for test determinism; keep only what an owned pipe/profile needs.
// Firefox is Playwright's own build, which needs Playwright's own flags.
- context = browser === "firefox"
+ context = kind === "agent" && browser !== "firefox"
+ ? (await agentChromium(browser ?? "chrome", directory)).contexts()[0]
+ : browser === "firefox"
? await firefox.launchPersistentContext(directory, { headless: false, viewport: null }).catch((error: Error) => {
if (!/Executable doesn't exist/.test(error.message)) throw error;
throw new Error("Playwright's Firefox is not installed on this machine. Ask the user to install it "
@@ -196,6 +321,7 @@ async function open({ browser, url, profile: kind, attach: target }: WebLaunchOp
// Playwright's default flags include this one: a page restored from the back-forward cache fires no load events.
"--disable-sync", "--disable-back-forward-cache", "--window-size=1100,800", "about:blank"],
});
+ if (!context) throw new Error("The launched browser exposed no default context.");
trackTabs(context);
active = tabs[0] ?? adopt(await context.newPage());
const owner = context.browser();
@@ -645,19 +771,45 @@ async function screenshot({ target, highlight, css }: { target?: WebTarget; high
}
async function close() {
- const owned = context, directory = profile, connected = attached;
- context = undefined; attached = undefined; active = undefined; profile = undefined; tabs.length = 0; downloads.length = 0; log.length = 0;
+ const owned = context, directory = profile, connected = attached, child = ownedProcess, verified = ownedVerified;
+ context = undefined; attached = undefined; ownedProcess = undefined; ownedVerified = false; approvalDirectory = undefined; approvalUrl = undefined; active = undefined; profile = undefined; tabs.length = 0; downloads.length = 0; log.length = 0;
+ if (child) {
+ // Only a browser whose CDP identity matches our child may receive Browser.close.
+ if (connected && verified) {
+ const cdp = await connected.newBrowserCDPSession().catch(() => undefined);
+ await cdp?.send("Browser.close").catch(() => undefined);
+ }
+ await connected?.close().catch(() => undefined);
+ if (child.pid && child.exitCode === null && child.signalCode === null) {
+ const exited = new Promise<void>(resolve => child.once("exit", () => resolve()));
+ if (connected && verified) await Promise.race([exited, new Promise<void>(resolve => setTimeout(resolve, 5_000))]);
+ if (child.exitCode === null && child.signalCode === null) {
+ if (!child.kill()) throw new Error("The owned browser could not be closed.");
+ await Promise.race([exited, new Promise<never>((_, reject) => setTimeout(() => reject(new Error("The owned browser did not exit.")), 5_000))]);
+ }
+ }
// Closing an attached browser only disconnects: the browser and its tabs stay open.
- if (connected) await connected.close();
+ } else if (connected) await connected.close();
else await owned?.close();
if (directory) await rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 });
+ if (profileGuard) {
+ const guard = profileGuard;
+ profileGuard = undefined;
+ if (guard.exitCode === null && guard.signalCode === null) {
+ const exited = new Promise<void>(resolve => guard.once("exit", () => resolve()));
+ guard.stdin?.end("\n");
+ await Promise.race([exited, new Promise<never>((_, reject) => setTimeout(() => reject(new Error("The agent profile lock did not release.")), 5_000))]);
+ }
+ }
return {};
}
-const operations: Record<string, (args: never) => Promise<unknown>> = { open, snapshot, act, check, screenshot, close };
+const operations: Record<string, (args: never) => Promise<unknown>> = { open, approve, snapshot, act, check, screenshot, close };
const input = readline.createInterface({ input: process.stdin, crlfDelay: Infinity });
+// A cancelled request may still be running. Never let close overtake an unfinished open or approval.
+let operationsTail: Promise<void> = Promise.resolve();
input.on("line", line => {
- void (async () => {
+ operationsTail = operationsTail.then(async () => {
let id: unknown;
try {
const request = JSON.parse(line) as { id: unknown; op: string; args: never };
@@ -669,10 +821,10 @@ input.on("line", line => {
const failure = error instanceof Error ? error : new Error(String(error));
process.stdout.write(JSON.stringify({ id, error: { name: failure.name, message: failure.message } }) + "\n");
}
- })();
+ });
});
input.on("close", () => {
- close().catch(error => {
+ operationsTail.then(() => close()).catch(error => {
process.stderr.write(`Owned browser cleanup failed: ${error instanceof Error ? error.message : String(error)}\n`);
process.exitCode = 1;
// An unfinished extension attach keeps Playwright's relay listening; the worker ends with its input either way.