Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
feat: computer_window names windows, computer_open sets WebView2 debugging (LPT-934)
alexshpunt-clanker[bot]da48b82ad10f
diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex 1846770..0f92562 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -876,6 +876,13 @@ export class ComputerResources {if (address.kind === "session") return this.closeAll();throw new ComputerRequestError("Delete closes a window, a sandbox or isolated session, a machine or computer:session. Observations, sequences, apps and displays need no closing.");}+ /** Name a window by its address, so computer:<name> resolves to it until it closes. The receipt names its machine. */+ async name(address: ComputerAddress, cwd: string, name: unknown, signal?: AbortSignal): Promise<ComputerResult> {+ if (address.kind !== "window") throw new ComputerRequestError("computer_window needs a window address from Search, Read or a name.");+ const place = this.place(address);+ await this.route(cwd, { window_ref: address.ref }, signal);+ return this.stamp(place, this.nameWindow(name, address.ref));+ }/*** Move, resize, minimize, maximize, focus or restore a window by its address: a fresh native state sample bounds the* one command, so no window-state address is handed out.diff --git a/src/computer-tools.ts b/src/computer-tools.tsindex b88c8f8..6069c49 100644--- a/src/computer-tools.ts+++ b/src/computer-tools.ts@@ -45,13 +45,17 @@ const openParameters = Type.Object({export: Type.Optional(Type.String({ description: "One host folder the sandbox can write to." })),}, { additionalProperties: false, description: "Open a disposable Windows Sandbox on that machine, apart from the user's desktop; pass {} for no shared folders." })),isolated_session: Type.Optional(Type.Boolean({ description: "Open an isolated agent session on that machine, with its own desktop and foreground input." })),+ webview2_debugging: Type.Optional(Type.Object({+ app: Type.String({ description: "The app's executable file name, such as ms-teams.exe." }),+ enabled: Type.Boolean({ description: "true opens the debugging port from the app's next start, whoever starts it; false turns it off again." }),+ }, { additionalProperties: false, description: "YOLO: set a WebView2 app's debugging port for its next start on that machine, so its window can be attached; opens nothing. Windows may ask the person there to approve the change." })),}, { additionalProperties: false });/** The Insert command that opens what computer_open asks for at this address. */export function openCommand(address: ComputerAddress, params: Static<typeof openParameters>): Record<string, unknown> {- const { app, path, args, debugging, browser, url, profile, sandbox, isolated_session } = params;+ const { app, path, args, debugging, browser, url, profile, sandbox, isolated_session, webview2_debugging } = params;const chosen = [app !== undefined && "app", path !== undefined && "path", (browser ?? url ?? profile) !== undefined && "browser",- sandbox !== undefined && "sandbox", isolated_session === true && "isolated_session"].filter(Boolean);+ sandbox !== undefined && "sandbox", isolated_session === true && "isolated_session", webview2_debugging !== undefined && "webview2_debugging"].filter(Boolean);if (chosen.length > 1) throw new ComputerRequestError(`Open one thing at a time: ${chosen.join(" and ")} were both given.`);if ((args !== undefined || debugging !== undefined) && path === undefined) throw new ComputerRequestError("args and debugging go with path.");const what = chosen[0];@@ -67,13 +71,15 @@ export function openCommand(address: ComputerAddress, params: Static<typeof openif (what === "browser") return { action: "open_browser", ...(browser ? { browser } : {}), ...(url ? { url } : {}), ...(profile ? { profile } : {}) };if (what === "sandbox") return { action: "open_sandbox", ...sandbox };if (what === "isolated_session") return { action: "open_session" };- throw new ComputerRequestError("Name what to open: app, path, browser or url, sandbox, or isolated_session.");+ if (what === "webview2_debugging") return { action: "webview2_debugging", ...webview2_debugging };+ throw new ComputerRequestError("Name what to open: app, path, browser or url, sandbox, isolated_session or webview2_debugging.");}const windowParameters = Type.Object({window: Type.String({ description: "Window address from Search, Read or a name, such as computer:zinpc/w3qk or computer:game." }),- action: Type.Union([Type.Literal("move"), Type.Literal("resize"), Type.Literal("maximize"), Type.Literal("minimize"), Type.Literal("focus"), Type.Literal("restore")],- { description: "focus activates a visible window; restore brings a minimized window back and activates it." }),+ action: Type.Union([Type.Literal("move"), Type.Literal("resize"), Type.Literal("maximize"), Type.Literal("minimize"), Type.Literal("focus"), Type.Literal("restore"), Type.Literal("name")],+ { description: "focus activates a visible window; restore brings a minimized window back and activates it; name gives the window a short address computer:<as>." }),+ as: Type.Optional(Type.String({ description: "name: 1-32 lowercase letters, digits, - or _, starting with a letter, such as game. It resolves until the window closes." })),x: Type.Optional(Type.Integer({ description: "move: new left edge in Windows desktop pixels." })),y: Type.Optional(Type.Integer({ description: "move: new top edge in Windows desktop pixels." })),width: Type.Optional(Type.Integer({ minimum: 1, description: "resize: new width in Windows desktop pixels." })),@@ -83,10 +89,11 @@ const windowParameters = Type.Object({/** The window management command of computer_window. */export function windowCommand(params: Static<typeof windowParameters>): Record<string, unknown> {const { window: _window, action, ...fields } = params;- const wanted = action === "move" ? ["x", "y"] : action === "resize" ? ["width", "height"] : [];+ const wanted = action === "move" ? ["x", "y"] : action === "resize" ? ["width", "height"] : action === "name" ? ["as"] : [];const given = Object.keys(fields);if (given.some(key => !wanted.includes(key)) || wanted.some(key => !given.includes(key)))- throw new ComputerRequestError(wanted.length ? `${action} needs ${wanted.join(" and ")} and nothing else.` : `${action} takes no coordinates or size.`);+ throw new ComputerRequestError(wanted.length ? `${action} needs ${wanted.join(" and ")} and nothing else.` : `${action} takes no coordinates, size or name.`);+ if (action === "name") return { action, ...fields };return { action, ...fields, ...(wanted.length ? { unit: "windows_desktop_pixels" } : {}) };}@@ -492,16 +499,16 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou});pi.registerTool({...common, name: "computer_window", label: "Computer window",- description: "Use computer_window to move, resize, maximize, minimize, focus or restore an app window by its address, without keys or clicks.",+ description: "Use computer_window to move, resize, maximize, minimize, focus or restore an app window by its address without keys or clicks, or to give it a short name.",parameters: windowParameters,annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },- ...rendered("computer_window", args => `${args.window} ${args.action}${args.action === "move" ? ` ${args.x},${args.y}` : args.action === "resize" ? ` ${args.width}x${args.height}` : ""}`),+ ...rendered("computer_window", args => `${args.window} ${args.action}${args.action === "move" ? ` ${args.x},${args.y}` : args.action === "resize" ? ` ${args.width}x${args.height}` : args.action === "name" ? ` ${args.as}` : ""}`),async execute(_id, params, signal, _update, ctx) {let command: Record<string, unknown> = {};return actionResult(await computerAction(resources, params.window, params.action, () => {command = windowCommand(params);return resources.address(params.window);- }, address => resources.manageWindow(address, ctx.cwd, command, signal)));+ }, address => command.action === "name" ? resources.name(address, ctx.cwd, command.as, signal) : resources.manageWindow(address, ctx.cwd, command, signal)));},});pi.registerTool({diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.tsindex 817b04f..370a4c8 100644--- a/test/computer-tools.test.ts+++ b/test/computer-tools.test.ts@@ -21,6 +21,8 @@ test("computer_open opens one thing, chosen by its fields and the address", () =expect(openCommand(machine, { at: "", browser: "msedge", profile: "agent" })).toEqual({ action: "open_browser", browser: "msedge", profile: "agent" });expect(openCommand(machine, { at: "", sandbox: { export: "C:\\out" } })).toEqual({ action: "open_sandbox", export: "C:\\out" });expect(openCommand(machine, { at: "", isolated_session: true })).toEqual({ action: "open_session" });+ expect(openCommand(machine, { at: "", webview2_debugging: { app: "ms-teams.exe", enabled: true } })).toEqual({ action: "webview2_debugging", app: "ms-teams.exe", enabled: true });+ expect(() => openCommand(machine, { at: "", path: "C:\\app.exe", webview2_debugging: { app: "app.exe", enabled: true } })).toThrow("path and webview2_debugging");expect(openCommand(at("computer:app/a1"), { at: "" })).toEqual({ action: "open" });expect(openCommand(at("computer:window/w1"), { at: "" })).toEqual({ action: "attach_browser" });expect(openCommand(at("computer:sandbox/s1"), { at: "", app: "Notepad" })).toEqual({ action: "open", app: "Notepad" });@@ -37,7 +39,10 @@ test("computer_window takes exactly the fields of its action", () => {expect(windowCommand({ window: "w", action: "resize", width: 800, height: 600 })).toEqual({ action: "resize", width: 800, height: 600, unit: "windows_desktop_pixels" });expect(windowCommand({ window: "w", action: "restore" })).toEqual({ action: "restore" });expect(() => windowCommand({ window: "w", action: "move", x: 1 })).toThrow("move needs x and y and nothing else.");- expect(() => windowCommand({ window: "w", action: "focus", width: 1 })).toThrow("focus takes no coordinates or size.");+ expect(() => windowCommand({ window: "w", action: "focus", width: 1 })).toThrow("focus takes no coordinates, size or name.");+ expect(windowCommand({ window: "w", action: "name", as: "game" })).toEqual({ action: "name", as: "game" });+ expect(() => windowCommand({ window: "w", action: "name" })).toThrow("name needs as and nothing else.");+ expect(() => windowCommand({ window: "w", action: "move", x: 1, y: 2, as: "game" })).toThrow("move needs x and y and nothing else.");});test("computer_wait maps to wait_for or a single verify sample", () => {