Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

A Remote Desktop display says its size comes from the client (LPT-953)

alexshpunt-clanker[bot]6c91c1d97354
diff --git a/src/display-mode.ts b/src/display-mode.ts
index be4a2f2..e047d16 100644
--- a/src/display-mode.ts
+++ b/src/display-mode.ts
@@ -15,8 +15,15 @@ export interface DisplayModes {
set(monitor: string, mode: DisplayMode, signal?: AbortSignal): Promise<DisplayModeChange>;
/** Return a device to an earlier mode. */
restore(device: string, mode: DisplayMode, signal?: AbortSignal): Promise<void>;
+ /** Whether the desktop is a Remote Desktop session now; its displays then take their size from the client. */
+ remoteSession(signal?: AbortSignal): Promise<boolean>;
}
+/** What the agent and the user can do about a Remote Desktop display's size. */
+export const REMOTE_DESKTOP_DISPLAY = "the resolution comes from the Remote Desktop client (its window size or its display settings), so computer_display cannot change it; " +
+ "the user can resize the client window or reconnect with another size, and set Windows scaling in Settings > System > Display. " +
+ "To read small controls, Read a region of the display or window in detail, or make a viewport with computer_view.";
+
const SCRIPT = String.raw`
Add-Type @"
using System; using System.Runtime.InteropServices;
@@ -78,6 +85,10 @@ export function displayModes(runner: CommandRunner): DisplayModes {
return {
set: (monitor, mode, signal) => change({ monitor, ...mode }, signal),
restore: async (device, mode, signal) => { await change({ device, ...mode }, signal); },
+ // SM_REMOTESESSION answers for the session as it is now; SESSIONNAME stays what it was when the shell started.
+ remoteSession: async signal => (await runner.run(powershellOf(runner), ["-NoLogo", "-NoProfile", "-NonInteractive", "-Command",
+ `Add-Type -Name M -Namespace PiDisplayRemote -MemberDefinition '[DllImport("user32.dll")] public static extern int GetSystemMetrics(int index);'; [PiDisplayRemote.M]::GetSystemMetrics(0x1000)`],
+ { signal, timeoutMs: 20_000 })).trim() !== "0",
};
}
diff --git a/src/provider.ts b/src/provider.ts
index 9ab1830..30972fd 100644
--- a/src/provider.ts
+++ b/src/provider.ts
@@ -7,6 +7,7 @@ import { connectAgentDocumentation } from "pi-agent-ide/api/documentation";
import { readFile } from "node:fs/promises";
import { ComputerResources, computerResultText, ComputerRequestError, FORCE, LOCAL_MACHINE, type SandboxFactory, type MachineFactory } from "./computer-resources.ts";
import { computerSource, displaySource, type ComputerAddress } from "./computer-address.ts";
+import { REMOTE_DESKTOP_DISPLAY } from "./display-mode.ts";
import type { WebLogEntry } from "./web-browser.ts";
import { diffIsEmpty, type PixelReport, type StateDiff } from "./successor.ts";
import type { PixelBox } from "./pixel-diff.ts";
@@ -140,6 +141,7 @@ export function presentComputerResult(result: ComputerResult, full = false): str
if (d.aria !== undefined) lines.push("Playwright aria snapshot; [ref=…] values are this observation's hints. Act with a target such as getByRole('button', { name: 'Save' }) on the window address, or a hint on this observation.", d.aria);
else if (!d.changes?.diff) for (const e of d.ui.elements) lines.push(`${e.hint} | ${label(e.role)} | ${label(e.name)} | ${label(e.value)} | enabled=${e.enabled} | image_bounds=${JSON.stringify(e.bounds.window_image ?? null)}`);
} else if (d.status === "listed") {
+ if (state.remote_session === true) lines.push(`These displays belong to a Remote Desktop session: ${REMOTE_DESKTOP_DISPLAY}`);
for (const kind of ["windows", "apps", "displays"] as const) {
const rows = state[kind];
if (!Array.isArray(rows)) continue;
diff --git a/src/session.ts b/src/session.ts
index d84e58e..6748917 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -20,7 +20,7 @@ import { screenRect, elementRegion, windowImageMapping, windowRegion, nativeClic
import { observationSource, sourceIdentity, type ObservationSource, type SourceIdentity, type NativePlatform } from "./observation-source.ts";
import { actionCapability, type ActionCapability } from "./action-capability.ts";
import { DEFAULT_MAX_LONG_EDGE, FULL_RESOLUTION, nativeImageCap, type CuaVisionSettings } from "./cua-image-settings.ts";
-import type { DisplayMode } from "./display-mode.ts";
+import { REMOTE_DESKTOP_DISPLAY, type DisplayMode } from "./display-mode.ts";
import { WEBVIEW2_DEBUGGING, type MachineHost, type ProcessHealth, type TextLine } from "./machine-host.ts";
import { setTimeout as delay } from "node:timers/promises";
import { cropDisplay, nativeRegion, validateDetail, type DetailContext, type DetailRequest } from "./native-detail.ts";
@@ -910,7 +910,10 @@ export class ComputerSession {
if (!this.driver.selectedDisplays) return this.result("refused", "Selected displays require an explicitly qualified Windows runtime.");
if (signal?.aborted) return this.result("refused", "Cancelled before display discovery.");
try {
+ // Remote Desktop is asked alongside the listing; when it cannot be told, the listing says nothing about it.
+ const remote = this.host?.displayModes.remoteSession(signal).catch(() => false) ?? Promise.resolve(false);
const state = await this.state("list_displays", {}, signal);
+ const remoteSession = await remote;
if (this.getMode() !== "yolo") return denied();
if (signal?.aborted) return this.result("refused", "Cancelled during display discovery; metadata withheld.");
if (state.platform !== "windows" || typeof state.topology_id !== "string" || !state.topology_id || !Array.isArray(state.displays) || !state.displays.length)
@@ -926,8 +929,9 @@ export class ComputerSession {
return { ...d, display_ref: entry.ref, source_identity: this.domained(sourceIdentity({ session_id: this.sessionId, display_ref: entry.ref, display_id: d.display_id, topology_id: topology }, d.display_id)) };
});
for (const ref of this.displayEntries.keys()) if (!current.has(ref)) this.displayEntries.delete(ref);
- return this.result("listed", "Native connected Windows displays. Read a returned display address for its pixels. Image pixels start at zero; bounds are host-global. No input authority.",
- { platform: "windows", topology_id: topology, displays: addressed });
+ return this.result("listed", "Native connected Windows displays. Read a returned display address for its pixels. Image pixels start at zero; bounds are host-global. No input authority." +
+ (remoteSession ? ` These displays belong to a Remote Desktop session: ${REMOTE_DESKTOP_DISPLAY}` : ""),
+ { platform: "windows", topology_id: topology, ...(remoteSession ? { remote_session: true } : {}), displays: addressed });
} catch (error) {
this.displayEntries.clear();
if (this.getMode() !== "yolo") return denied();
@@ -1043,7 +1047,9 @@ export class ComputerSession {
catch (error) {
// A cancelled or timed-out change may still have reached Windows.
if (signal?.aborted) return this.result("unknown", "Cancelled during the display change; the resolution may have changed. Read the displays again.");
- return this.result("refused", `Display not changed. ${error instanceof Error ? error.message : error}`);
+ const remote = await this.host.displayModes.remoteSession(signal).catch(() => false);
+ return this.result("refused", `Display not changed. ${error instanceof Error ? error.message : error}` +
+ (remote ? ` This display belongs to a Remote Desktop session: ${REMOTE_DESKTOP_DISPLAY}` : ""));
}
if (!this.replacedModes.has(change.device)) this.replacedModes.set(change.device, change.previous);
const original = this.replacedModes.get(change.device)!;