Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
A Remote Desktop display says its size comes from the client (LPT-953)
alexshpunt-clanker[bot]6c91c1d97354
diff --git a/src/display-mode.ts b/src/display-mode.tsindex be4a2f2..e047d16 100644--- a/src/display-mode.ts+++ b/src/display-mode.ts@@ -15,8 +15,15 @@ export interface DisplayModes {set(monitor: string, mode: DisplayMode, signal?: AbortSignal): Promise<DisplayModeChange>;/** Return a device to an earlier mode. */restore(device: string, mode: DisplayMode, signal?: AbortSignal): Promise<void>;+ /** Whether the desktop is a Remote Desktop session now; its displays then take their size from the client. */+ remoteSession(signal?: AbortSignal): Promise<boolean>;}+/** What the agent and the user can do about a Remote Desktop display's size. */+export const REMOTE_DESKTOP_DISPLAY = "the resolution comes from the Remote Desktop client (its window size or its display settings), so computer_display cannot change it; " ++ "the user can resize the client window or reconnect with another size, and set Windows scaling in Settings > System > Display. " ++ "To read small controls, Read a region of the display or window in detail, or make a viewport with computer_view.";+const SCRIPT = String.raw`Add-Type @"using System; using System.Runtime.InteropServices;@@ -78,6 +85,10 @@ export function displayModes(runner: CommandRunner): DisplayModes {return {set: (monitor, mode, signal) => change({ monitor, ...mode }, signal),restore: async (device, mode, signal) => { await change({ device, ...mode }, signal); },+ // SM_REMOTESESSION answers for the session as it is now; SESSIONNAME stays what it was when the shell started.+ remoteSession: async signal => (await runner.run(powershellOf(runner), ["-NoLogo", "-NoProfile", "-NonInteractive", "-Command",+ `Add-Type -Name M -Namespace PiDisplayRemote -MemberDefinition '[DllImport("user32.dll")] public static extern int GetSystemMetrics(int index);'; [PiDisplayRemote.M]::GetSystemMetrics(0x1000)`],+ { signal, timeoutMs: 20_000 })).trim() !== "0",};}diff --git a/src/provider.ts b/src/provider.tsindex 9ab1830..30972fd 100644--- a/src/provider.ts+++ b/src/provider.ts@@ -7,6 +7,7 @@ import { connectAgentDocumentation } from "pi-agent-ide/api/documentation";import { readFile } from "node:fs/promises";import { ComputerResources, computerResultText, ComputerRequestError, FORCE, LOCAL_MACHINE, type SandboxFactory, type MachineFactory } from "./computer-resources.ts";import { computerSource, displaySource, type ComputerAddress } from "./computer-address.ts";+import { REMOTE_DESKTOP_DISPLAY } from "./display-mode.ts";import type { WebLogEntry } from "./web-browser.ts";import { diffIsEmpty, type PixelReport, type StateDiff } from "./successor.ts";import type { PixelBox } from "./pixel-diff.ts";@@ -140,6 +141,7 @@ export function presentComputerResult(result: ComputerResult, full = false): strif (d.aria !== undefined) lines.push("Playwright aria snapshot; [ref=…] values are this observation's hints. Act with a target such as getByRole('button', { name: 'Save' }) on the window address, or a hint on this observation.", d.aria);else if (!d.changes?.diff) for (const e of d.ui.elements) lines.push(`${e.hint} | ${label(e.role)} | ${label(e.name)} | ${label(e.value)} | enabled=${e.enabled} | image_bounds=${JSON.stringify(e.bounds.window_image ?? null)}`);} else if (d.status === "listed") {+ if (state.remote_session === true) lines.push(`These displays belong to a Remote Desktop session: ${REMOTE_DESKTOP_DISPLAY}`);for (const kind of ["windows", "apps", "displays"] as const) {const rows = state[kind];if (!Array.isArray(rows)) continue;diff --git a/src/session.ts b/src/session.tsindex d84e58e..6748917 100644--- a/src/session.ts+++ b/src/session.ts@@ -20,7 +20,7 @@ import { screenRect, elementRegion, windowImageMapping, windowRegion, nativeClicimport { observationSource, sourceIdentity, type ObservationSource, type SourceIdentity, type NativePlatform } from "./observation-source.ts";import { actionCapability, type ActionCapability } from "./action-capability.ts";import { DEFAULT_MAX_LONG_EDGE, FULL_RESOLUTION, nativeImageCap, type CuaVisionSettings } from "./cua-image-settings.ts";-import type { DisplayMode } from "./display-mode.ts";+import { REMOTE_DESKTOP_DISPLAY, type DisplayMode } from "./display-mode.ts";import { WEBVIEW2_DEBUGGING, type MachineHost, type ProcessHealth, type TextLine } from "./machine-host.ts";import { setTimeout as delay } from "node:timers/promises";import { cropDisplay, nativeRegion, validateDetail, type DetailContext, type DetailRequest } from "./native-detail.ts";@@ -910,7 +910,10 @@ export class ComputerSession {if (!this.driver.selectedDisplays) return this.result("refused", "Selected displays require an explicitly qualified Windows runtime.");if (signal?.aborted) return this.result("refused", "Cancelled before display discovery.");try {+ // Remote Desktop is asked alongside the listing; when it cannot be told, the listing says nothing about it.+ const remote = this.host?.displayModes.remoteSession(signal).catch(() => false) ?? Promise.resolve(false);const state = await this.state("list_displays", {}, signal);+ const remoteSession = await remote;if (this.getMode() !== "yolo") return denied();if (signal?.aborted) return this.result("refused", "Cancelled during display discovery; metadata withheld.");if (state.platform !== "windows" || typeof state.topology_id !== "string" || !state.topology_id || !Array.isArray(state.displays) || !state.displays.length)@@ -926,8 +929,9 @@ export class ComputerSession {return { ...d, display_ref: entry.ref, source_identity: this.domained(sourceIdentity({ session_id: this.sessionId, display_ref: entry.ref, display_id: d.display_id, topology_id: topology }, d.display_id)) };});for (const ref of this.displayEntries.keys()) if (!current.has(ref)) this.displayEntries.delete(ref);- return this.result("listed", "Native connected Windows displays. Read a returned display address for its pixels. Image pixels start at zero; bounds are host-global. No input authority.",- { platform: "windows", topology_id: topology, displays: addressed });+ return this.result("listed", "Native connected Windows displays. Read a returned display address for its pixels. Image pixels start at zero; bounds are host-global. No input authority." ++ (remoteSession ? ` These displays belong to a Remote Desktop session: ${REMOTE_DESKTOP_DISPLAY}` : ""),+ { platform: "windows", topology_id: topology, ...(remoteSession ? { remote_session: true } : {}), displays: addressed });} catch (error) {this.displayEntries.clear();if (this.getMode() !== "yolo") return denied();@@ -1043,7 +1047,9 @@ export class ComputerSession {catch (error) {// A cancelled or timed-out change may still have reached Windows.if (signal?.aborted) return this.result("unknown", "Cancelled during the display change; the resolution may have changed. Read the displays again.");- return this.result("refused", `Display not changed. ${error instanceof Error ? error.message : error}`);+ const remote = await this.host.displayModes.remoteSession(signal).catch(() => false);+ return this.result("refused", `Display not changed. ${error instanceof Error ? error.message : error}` ++ (remote ? ` This display belongs to a Remote Desktop session: ${REMOTE_DESKTOP_DISPLAY}` : ""));}if (!this.replacedModes.has(change.device)) this.replacedModes.set(change.device, change.previous);const original = this.replacedModes.get(change.device)!;