Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

7.2 KB
Raw Blame
import { powershellOf, type CommandRunner } from "./runner.ts";

/** A display resolution in physical pixels. */
export interface DisplayMode { width: number; height: number }
/** A mode change on one GDI display device, with the mode it replaced. */
export interface DisplayModeChange { device: string; previous: DisplayMode; current: DisplayMode }

/**
 * Change a Windows display's resolution through the runner's own Windows (this host or a machine over SSH).
 * Changes are dynamic: Windows does not save them, so a sign-out or restart also returns the saved mode.
 * CUA has no display-mode tool; this does no capture or input.
 */
export interface DisplayModes {
  /** Switch the monitor with this native HMONITOR to a mode the driver lists, keeping colour depth and preferring the current refresh rate. */
  set(monitor: string, mode: DisplayMode, signal?: AbortSignal): Promise<DisplayModeChange>;
  /** Return a device to an earlier mode. */
  restore(device: string, mode: DisplayMode, signal?: AbortSignal): Promise<void>;
  /** Whether the desktop is a Remote Desktop session now; its displays then take their size from the client. */
  remoteSession(signal?: AbortSignal): Promise<boolean>;
}

/** What the agent and the user can do about a Remote Desktop display's size. */
export const REMOTE_DESKTOP_DISPLAY = "the resolution comes from the Remote Desktop client (its window size or its display settings), so computer_display cannot change it; " +
  "the user can resize the client window or reconnect with another size, and set Windows scaling in Settings > System > Display. " +
  "To read small controls, Read a region of the display or window in detail, or make a viewport with computer_view.";

const SCRIPT = String.raw`
Add-Type @"
using System; using System.Runtime.InteropServices;
public static class PiDisplay {
  [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
  public struct DEVMODE {
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 32)] public string dmDeviceName;
    public short dmSpecVersion, dmDriverVersion, dmSize, dmDriverExtra; public int dmFields;
    public int dmPositionX, dmPositionY, dmDisplayOrientation, dmDisplayFixedOutput;
    public short dmColor, dmDuplex, dmYResolution, dmTTOption, dmCollate;
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 32)] public string dmFormName;
    public short dmLogPixels; public int dmBitsPerPel, dmPelsWidth, dmPelsHeight, dmDisplayFlags, dmDisplayFrequency;
    public int dmICMMethod, dmICMIntent, dmMediaType, dmDitherType, dmReserved1, dmReserved2, dmPanningWidth, dmPanningHeight;
  }
  [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
  public struct MONITORINFOEX { public int cbSize; public int l, t, r, b, wl, wt, wr, wb; public int dwFlags;
    [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 32)] public string szDevice; }
  [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern bool GetMonitorInfo(IntPtr monitor, ref MONITORINFOEX info);
  [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern bool EnumDisplaySettings(string device, int mode, ref DEVMODE dm);
  [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int ChangeDisplaySettingsEx(string device, ref DEVMODE dm, IntPtr hwnd, int flags, IntPtr param);
  public static DEVMODE Blank() { var dm = new DEVMODE(); dm.dmSize = (short)Marshal.SizeOf(typeof(DEVMODE)); return dm; }
}
"@
$request = [Console]::In.ReadToEnd() | ConvertFrom-Json
$device = $request.device
if ($request.monitor) {
  $info = New-Object PiDisplay+MONITORINFOEX; $info.cbSize = [Runtime.InteropServices.Marshal]::SizeOf($info)
  if (-not [PiDisplay]::GetMonitorInfo([IntPtr][Int64]$request.monitor, [ref]$info)) { throw "The monitor is gone. Read the displays again." }
  $device = $info.szDevice
}
$current = [PiDisplay]::Blank()
if (-not [PiDisplay]::EnumDisplaySettings($device, -1, [ref]$current)) { throw "Cannot read the current mode of $device." }
$best = $null; $i = 0; $mode = [PiDisplay]::Blank()
while ([PiDisplay]::EnumDisplaySettings($device, $i, [ref]$mode)) {
  if ($mode.dmPelsWidth -eq $request.width -and $mode.dmPelsHeight -eq $request.height -and $mode.dmBitsPerPel -eq $current.dmBitsPerPel -and
      ($best -eq $null -or $mode.dmDisplayFrequency -eq $current.dmDisplayFrequency -or
       ($best.dmDisplayFrequency -ne $current.dmDisplayFrequency -and $mode.dmDisplayFrequency -gt $best.dmDisplayFrequency))) { $best = $mode }
  $i++; $mode = [PiDisplay]::Blank()
}
if ($best -eq $null) {
  $sizes = @{}; $i = 0; $mode = [PiDisplay]::Blank()
  while ([PiDisplay]::EnumDisplaySettings($device, $i, [ref]$mode)) { if ($mode.dmBitsPerPel -eq $current.dmBitsPerPel) { $sizes["$($mode.dmPelsWidth)x$($mode.dmPelsHeight)"] = 1 }; $i++; $mode = [PiDisplay]::Blank() }
  throw "$device has no $($request.width)x$($request.height) mode. Supported: $(($sizes.Keys | Sort-Object { [int]($_ -split 'x')[0] }, { [int]($_ -split 'x')[1] }) -join ', ')."
}
$best.dmFields = 0x5C0000
$status = [PiDisplay]::ChangeDisplaySettingsEx($device, [ref]$best, [IntPtr]::Zero, 0, [IntPtr]::Zero)
if ($status -ne 0) { throw "Windows refused the mode change (code $status)." }
@{ device = $device; previous = @{ width = $current.dmPelsWidth; height = $current.dmPelsHeight }; current = @{ width = $best.dmPelsWidth; height = $best.dmPelsHeight } } | ConvertTo-Json -Compress
`;

/** Display modes changed through Windows PowerShell started by the runner. */
export function displayModes(runner: CommandRunner): DisplayModes {
  const change = async (request: Record<string, unknown>, signal?: AbortSignal): Promise<DisplayModeChange> => {
    const output = await runner.run(powershellOf(runner), ["-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-EncodedCommand",
      Buffer.from(`$ProgressPreference = 'SilentlyContinue'; $ErrorActionPreference = 'Stop'\ntry {\n${SCRIPT}\n} catch { [Console]::Error.WriteLine($_.Exception.Message); exit 1 }`, "utf16le").toString("base64")],
      { input: JSON.stringify(request), signal, timeoutMs: 30_000 });
    return JSON.parse(output.trim()) as DisplayModeChange;
  };
  return {
    set: (monitor, mode, signal) => change({ monitor, ...mode }, signal),
    restore: async (device, mode, signal) => { await change({ device, ...mode }, signal); },
    // SM_REMOTESESSION answers for the session as it is now; SESSIONNAME stays what it was when the shell started.
    remoteSession: async signal => (await runner.run(powershellOf(runner), ["-NoLogo", "-NoProfile", "-NonInteractive", "-Command",
      `Add-Type -Name M -Namespace PiDisplayRemote -MemberDefinition '[DllImport("user32.dll")] public static extern int GetSystemMetrics(int index);'; [PiDisplayRemote.M]::GetSystemMetrics(0x1000)`],
      { signal, timeoutMs: 20_000 })).trim() !== "0",
  };
}

/** Accept only whole positive pixel sizes a display could have. */
export function displayMode(value: Record<string, unknown>): DisplayMode {
  const { width, height } = value;
  if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height) || (width as number) < 640 || (height as number) < 480 || (width as number) > 16384 || (height as number) > 16384)
    throw new Error("Resize needs integer width and height in physical pixels, at least 640x480.");
  return { width: width as number, height: height as number };
}