Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

feat: computer_window names windows, computer_open sets WebView2 debugging (LPT-934)

alexshpunt-clanker[bot]da48b82ad10f
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index 1846770..0f92562 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -876,6 +876,13 @@ export class ComputerResources {
if (address.kind === "session") return this.closeAll();
throw new ComputerRequestError("Delete closes a window, a sandbox or isolated session, a machine or computer:session. Observations, sequences, apps and displays need no closing.");
}
+ /** Name a window by its address, so computer:<name> resolves to it until it closes. The receipt names its machine. */
+ async name(address: ComputerAddress, cwd: string, name: unknown, signal?: AbortSignal): Promise<ComputerResult> {
+ if (address.kind !== "window") throw new ComputerRequestError("computer_window needs a window address from Search, Read or a name.");
+ const place = this.place(address);
+ await this.route(cwd, { window_ref: address.ref }, signal);
+ return this.stamp(place, this.nameWindow(name, address.ref));
+ }
/**
* Move, resize, minimize, maximize, focus or restore a window by its address: a fresh native state sample bounds the
* one command, so no window-state address is handed out.
diff --git a/src/computer-tools.ts b/src/computer-tools.ts
index b88c8f8..6069c49 100644
--- a/src/computer-tools.ts
+++ b/src/computer-tools.ts
@@ -45,13 +45,17 @@ const openParameters = Type.Object({
export: Type.Optional(Type.String({ description: "One host folder the sandbox can write to." })),
}, { additionalProperties: false, description: "Open a disposable Windows Sandbox on that machine, apart from the user's desktop; pass {} for no shared folders." })),
isolated_session: Type.Optional(Type.Boolean({ description: "Open an isolated agent session on that machine, with its own desktop and foreground input." })),
+ webview2_debugging: Type.Optional(Type.Object({
+ app: Type.String({ description: "The app's executable file name, such as ms-teams.exe." }),
+ enabled: Type.Boolean({ description: "true opens the debugging port from the app's next start, whoever starts it; false turns it off again." }),
+ }, { additionalProperties: false, description: "YOLO: set a WebView2 app's debugging port for its next start on that machine, so its window can be attached; opens nothing. Windows may ask the person there to approve the change." })),
}, { additionalProperties: false });
/** The Insert command that opens what computer_open asks for at this address. */
export function openCommand(address: ComputerAddress, params: Static<typeof openParameters>): Record<string, unknown> {
- const { app, path, args, debugging, browser, url, profile, sandbox, isolated_session } = params;
+ const { app, path, args, debugging, browser, url, profile, sandbox, isolated_session, webview2_debugging } = params;
const chosen = [app !== undefined && "app", path !== undefined && "path", (browser ?? url ?? profile) !== undefined && "browser",
- sandbox !== undefined && "sandbox", isolated_session === true && "isolated_session"].filter(Boolean);
+ sandbox !== undefined && "sandbox", isolated_session === true && "isolated_session", webview2_debugging !== undefined && "webview2_debugging"].filter(Boolean);
if (chosen.length > 1) throw new ComputerRequestError(`Open one thing at a time: ${chosen.join(" and ")} were both given.`);
if ((args !== undefined || debugging !== undefined) && path === undefined) throw new ComputerRequestError("args and debugging go with path.");
const what = chosen[0];
@@ -67,13 +71,15 @@ export function openCommand(address: ComputerAddress, params: Static<typeof open
if (what === "browser") return { action: "open_browser", ...(browser ? { browser } : {}), ...(url ? { url } : {}), ...(profile ? { profile } : {}) };
if (what === "sandbox") return { action: "open_sandbox", ...sandbox };
if (what === "isolated_session") return { action: "open_session" };
- throw new ComputerRequestError("Name what to open: app, path, browser or url, sandbox, or isolated_session.");
+ if (what === "webview2_debugging") return { action: "webview2_debugging", ...webview2_debugging };
+ throw new ComputerRequestError("Name what to open: app, path, browser or url, sandbox, isolated_session or webview2_debugging.");
}
const windowParameters = Type.Object({
window: Type.String({ description: "Window address from Search, Read or a name, such as computer:zinpc/w3qk or computer:game." }),
- action: Type.Union([Type.Literal("move"), Type.Literal("resize"), Type.Literal("maximize"), Type.Literal("minimize"), Type.Literal("focus"), Type.Literal("restore")],
- { description: "focus activates a visible window; restore brings a minimized window back and activates it." }),
+ action: Type.Union([Type.Literal("move"), Type.Literal("resize"), Type.Literal("maximize"), Type.Literal("minimize"), Type.Literal("focus"), Type.Literal("restore"), Type.Literal("name")],
+ { description: "focus activates a visible window; restore brings a minimized window back and activates it; name gives the window a short address computer:<as>." }),
+ as: Type.Optional(Type.String({ description: "name: 1-32 lowercase letters, digits, - or _, starting with a letter, such as game. It resolves until the window closes." })),
x: Type.Optional(Type.Integer({ description: "move: new left edge in Windows desktop pixels." })),
y: Type.Optional(Type.Integer({ description: "move: new top edge in Windows desktop pixels." })),
width: Type.Optional(Type.Integer({ minimum: 1, description: "resize: new width in Windows desktop pixels." })),
@@ -83,10 +89,11 @@ const windowParameters = Type.Object({
/** The window management command of computer_window. */
export function windowCommand(params: Static<typeof windowParameters>): Record<string, unknown> {
const { window: _window, action, ...fields } = params;
- const wanted = action === "move" ? ["x", "y"] : action === "resize" ? ["width", "height"] : [];
+ const wanted = action === "move" ? ["x", "y"] : action === "resize" ? ["width", "height"] : action === "name" ? ["as"] : [];
const given = Object.keys(fields);
if (given.some(key => !wanted.includes(key)) || wanted.some(key => !given.includes(key)))
- throw new ComputerRequestError(wanted.length ? `${action} needs ${wanted.join(" and ")} and nothing else.` : `${action} takes no coordinates or size.`);
+ throw new ComputerRequestError(wanted.length ? `${action} needs ${wanted.join(" and ")} and nothing else.` : `${action} takes no coordinates, size or name.`);
+ if (action === "name") return { action, ...fields };
return { action, ...fields, ...(wanted.length ? { unit: "windows_desktop_pixels" } : {}) };
}
@@ -492,16 +499,16 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
});
pi.registerTool({
...common, name: "computer_window", label: "Computer window",
- description: "Use computer_window to move, resize, maximize, minimize, focus or restore an app window by its address, without keys or clicks.",
+ description: "Use computer_window to move, resize, maximize, minimize, focus or restore an app window by its address without keys or clicks, or to give it a short name.",
parameters: windowParameters,
annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
- ...rendered("computer_window", args => `${args.window} ${args.action}${args.action === "move" ? ` ${args.x},${args.y}` : args.action === "resize" ? ` ${args.width}x${args.height}` : ""}`),
+ ...rendered("computer_window", args => `${args.window} ${args.action}${args.action === "move" ? ` ${args.x},${args.y}` : args.action === "resize" ? ` ${args.width}x${args.height}` : args.action === "name" ? ` ${args.as}` : ""}`),
async execute(_id, params, signal, _update, ctx) {
let command: Record<string, unknown> = {};
return actionResult(await computerAction(resources, params.window, params.action, () => {
command = windowCommand(params);
return resources.address(params.window);
- }, address => resources.manageWindow(address, ctx.cwd, command, signal)));
+ }, address => command.action === "name" ? resources.name(address, ctx.cwd, command.as, signal) : resources.manageWindow(address, ctx.cwd, command, signal)));
},
});
pi.registerTool({
diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.ts
index 817b04f..370a4c8 100644
--- a/test/computer-tools.test.ts
+++ b/test/computer-tools.test.ts
@@ -21,6 +21,8 @@ test("computer_open opens one thing, chosen by its fields and the address", () =
expect(openCommand(machine, { at: "", browser: "msedge", profile: "agent" })).toEqual({ action: "open_browser", browser: "msedge", profile: "agent" });
expect(openCommand(machine, { at: "", sandbox: { export: "C:\\out" } })).toEqual({ action: "open_sandbox", export: "C:\\out" });
expect(openCommand(machine, { at: "", isolated_session: true })).toEqual({ action: "open_session" });
+ expect(openCommand(machine, { at: "", webview2_debugging: { app: "ms-teams.exe", enabled: true } })).toEqual({ action: "webview2_debugging", app: "ms-teams.exe", enabled: true });
+ expect(() => openCommand(machine, { at: "", path: "C:\\app.exe", webview2_debugging: { app: "app.exe", enabled: true } })).toThrow("path and webview2_debugging");
expect(openCommand(at("computer:app/a1"), { at: "" })).toEqual({ action: "open" });
expect(openCommand(at("computer:window/w1"), { at: "" })).toEqual({ action: "attach_browser" });
expect(openCommand(at("computer:sandbox/s1"), { at: "", app: "Notepad" })).toEqual({ action: "open", app: "Notepad" });
@@ -37,7 +39,10 @@ test("computer_window takes exactly the fields of its action", () => {
expect(windowCommand({ window: "w", action: "resize", width: 800, height: 600 })).toEqual({ action: "resize", width: 800, height: 600, unit: "windows_desktop_pixels" });
expect(windowCommand({ window: "w", action: "restore" })).toEqual({ action: "restore" });
expect(() => windowCommand({ window: "w", action: "move", x: 1 })).toThrow("move needs x and y and nothing else.");
- expect(() => windowCommand({ window: "w", action: "focus", width: 1 })).toThrow("focus takes no coordinates or size.");
+ expect(() => windowCommand({ window: "w", action: "focus", width: 1 })).toThrow("focus takes no coordinates, size or name.");
+ expect(windowCommand({ window: "w", action: "name", as: "game" })).toEqual({ action: "name", as: "game" });
+ expect(() => windowCommand({ window: "w", action: "name" })).toThrow("name needs as and nothing else.");
+ expect(() => windowCommand({ window: "w", action: "move", x: 1, y: 2, as: "game" })).toThrow("move needs x and y and nothing else.");
});
test("computer_wait maps to wait_for or a single verify sample", () => {