Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

7.8 KB
Raw Blame
import { afterAll, expect, test } from "vitest";
import { connectLive, nativeForm, openWindow, text, WINDOWS_POWERSHELL_PATH as POWERSHELL } from "./remote.ts";
import { computerSource } from "../../src/computer-address.ts";

// LPT-626: strict UIA actions must not take the foreground or move their target to another virtual desktop.
// Everything runs in a disposable Windows Sandbox: a sentinel console holds the foreground, a minimized sampler
// reads the foreground window every 20 ms and reports which titles took it, and a second virtual desktop is
// created inside the guest only.
const live = await connectLive();
let sandbox: string | undefined;
afterAll(async () => { if (sandbox) await live.insert(sandbox, { action: "close" }).catch(() => undefined); await live.close(); });
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));

const PAGE = `<title>Semantic targets</title><button onclick="this.textContent='Count '+(++window.n||(window.n=1))">Count 0</button>
<select size=3 aria-label=Pick><option>alpha<option>beta<option>gamma</select><input aria-label=Field>`;
const SAMPLER = `Add-Type -Name U -Namespace W -MemberDefinition '[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll", CharSet=CharSet.Unicode)] public static extern int GetWindowText(IntPtr h, System.Text.StringBuilder s, int n);'
$sb = New-Object System.Text.StringBuilder 256; $n = 0; $who = @{}; $Host.UI.RawUI.WindowTitle = 'FGPROBE n=0 who='
$seen = $false
while ($true) { $h = [W.U]::GetForegroundWindow(); [void]$sb.Clear(); [void][W.U]::GetWindowText($h, $sb, 256); $t = $sb.ToString(); $n++
  if ($t -like '*SENTINEL*') { $seen = $true } elseif ($seen) { $k = ($t -replace '[^A-Za-z0-9 ]', '_'); if ($k.Length -gt 40) { $k = $k.Substring($k.Length - 40) }; $who[$k] = 1 + [int]$who[$k] }
  if ($n % 10 -eq 0) { $Host.UI.RawUI.WindowTitle = 'FGPROBE n=' + $n + ' who=' + (($who.GetEnumerator() | ForEach-Object { $_.Key + ':' + $_.Value }) -join ';') }
  Start-Sleep -Milliseconds 20 }`;
const encoded = (script: string) => Buffer.from(script, "utf16le").toString("base64");
const CMD = "C:\\Windows\\System32\\cmd.exe";
type Element = { role?: string | null; name?: string | null; hint: string; value?: unknown; selected?: unknown };

test("strict UIA actions keep the sentinel foreground and the target's desktop", async () => {
  const opened = await live.insert(live.machine, { action: "open_sandbox" });
  expect(opened.details.status, text(opened)).toBe("delivered");
  sandbox = computerSource("sandbox", opened.details.sandbox_ref!);
  const open = (path: string, args: string[], title: string) => openWindow(live, sandbox!, path, args, title);
  const edge = await open("C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
    ["--no-first-run", "--no-default-browser-check", "--new-window", `data:text/html,${encodeURIComponent(PAGE)}`], "Semantic targets");
  // The guest has neither Notepad (its System32 stub needs the Store app) nor Character Map.
  const native = await open(POWERSHELL, nativeForm("NATIVE TARGET"), "NATIVE TARGET");
  const sentinel = await open(CMD, ["/k", "title SENTINEL"], "SENTINEL");
  await open(POWERSHELL, ["-NoProfile", "-WindowStyle", "Minimized", "-EncodedCommand", encoded(SAMPLER)], "FGPROBE");

  const thieves = async () => {
    await sleep(600);
    const who = JSON.stringify((await live.search(sandbox!, "FGPROBE")).details.state).match(/FGPROBE n=\d+ who=([^"\\]*)/)?.[1] ?? "";
    return who.split(";").filter(Boolean);
  };
  const ui = async (window: string) => {
    const view = await live.read(window, ["ui"]);
    if (!view.details.ui) {
      const ref = window.split("/").pop();
      const listed = (JSON.parse(JSON.stringify((await live.search(sandbox!, "regex:.")).details.state)).windows as { window_ref?: string; title?: string; app_name?: string; minimized?: boolean; is_on_screen?: boolean }[])
        .map(w => `${w.window_ref === ref ? "*" : ""}${w.app_name}|${w.title}|min=${w.minimized}|on=${w.is_on_screen}`);
      throw new Error(`No UI for ${window}: STATE=${JSON.stringify(view.details.state)} WINDOWS=${JSON.stringify(listed)}`.replace(/\n/g, " "));
    }
    const desktop = view.details.source_identity?.virtual_desktop;
    return { view, elements: view.details.ui.elements as Element[],
      desktop: desktop?.status === "sampled" ? { desktop_id: desktop.desktop_id, on_current_desktop: desktop.on_current_desktop } : desktop };
  };
  const act = async (window: string, match: (e: Element) => boolean, action: Record<string, unknown>) => {
    const { view, elements } = await ui(window);
    const element = elements.find(match);
    // An app can expose only its frame through UIA while on another virtual desktop; then nothing is dispatched.
    if (!element) return "absent";
    const done = await live.insert(`computer:observation/${view.details.observation_id}?hint=${encodeURIComponent(element.hint)}`, action);
    return done.details.status === "delivered" ? "delivered" : String((done.details.state as { code?: string } | undefined)?.code ?? done.details.status);
  };
  const button = (e: Element) => e.role === "Button" && /^Count/.test(e.name ?? "");
  const item = (name: string) => (e: Element) => /ListItem/.test(e.role ?? "") && e.name === name;
  const edit = (e: Element) => e.role === "Edit";
  const round = async (label: string) => {
    const edgeDesktop = (await ui(edge)).desktop, nativeDesktop = (await ui(native)).desktop;
    const before = await thieves();
    const outcomes = {
      edge: [await act(edge, button, { action: "invoke" }), await act(edge, item("beta"), { action: "select" }), await act(edge, edit, { action: "set_value", text: label })],
      native: [await act(native, button, { action: "invoke" }), await act(native, item("gamma"), { action: "select" }), await act(native, edit, { action: "set_value", text: label })],
    };
    const after = await thieves();
    const readback = await ui(native);
    console.log(label, JSON.stringify({ outcomes, before, after }));
    // Neither target ever took the foreground.
    expect(after.filter(entry => /NATIVE TARGET|Semantic targets/.test(entry))).toEqual([]);
    // Each target stays on the desktop it started on.
    expect((await ui(edge)).desktop).toEqual(edgeDesktop);
    expect(readback.desktop).toEqual(nativeDesktop);
    return { outcomes, readback: readback.elements };
  };

  const focus = async (window: string) => {
    expect((await live.resources.manageWindow(live.resources.address(window), live.cwd, { action: "focus" })).details.status).toBe("delivered");
    await sleep(1000);
  };
  await focus(sentinel);
  const current = await round("current");
  // Edge reports its page elements under another process, so the driver refuses them before any call.
  expect(current.outcomes.edge).toEqual(Array(3).fill("semantic_element_process_changed"));
  expect(current.outcomes.native).toEqual(["delivered", "delivered", "delivered"]);
  expect(current.readback.find(button)?.name).toBe("Count 1");
  expect(current.readback.find(item("gamma"))?.selected).toBe(true);
  expect(current.readback.find(edit)?.value).toBe("current");

  // A second virtual desktop inside the guest; both targets stay behind on the first.
  const view = await live.read(sentinel, ["image"]);
  const created = await live.insert(`computer:observation/${view.details.observation_id}?delivery=foreground`, { action: "hotkey", keys: ["ctrl", "win", "d"] });
  expect(created.details.status).toBe("delivered");
  await sleep(1500);
  await open(CMD, ["/k", "title SENTINEL2"], "SENTINEL2");
  expect((await ui(native)).desktop).toMatchObject({ on_current_desktop: false });
  const other = await round("other");
  // The WinForms target exposes only its frame there, so its content actions are not dispatched.
  expect(other.outcomes.native).toEqual(["absent", "absent", "absent"]);
});