Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

3.7 KB
Raw Blame
import { mkdir, writeFile } from "node:fs/promises";
import { afterAll, expect, test } from "vitest";
import { connectLive, paintTool, text } from "./remote.ts";
import { presentComputerResult } from "../../src/provider.ts";

// LPT-915: an action on a canvas reports the changed area of the window's pixels with before/after crops.
const live = await connectLive();
const opened: number[] = [];
afterAll(async () => { for (const pid of opened) await live.kill(pid); await live.close(); });

test("a Paint stroke reports a box around it with crops, and a click on empty space reports no change", async () => {
  const paint = await live.insert(live.machine, { action: "open", app: "Paint" });
  expect(paint.details.status, text(paint)).toBe("observed");
  opened.push(paint.details.target!.pid);
  const window = `computer:window/${paint.details.window_ref}`;
  await paintTool(live, window, "Pencil");

  const view = await live.read(window, ["full"]);
  const canvas = view.details.ui!.elements.find(e => e.role === "Group" && /on Canvas$/.test(e.name ?? ""))!.bounds.window_image!;
  const from = { x: canvas.x + Math.round(canvas.width * 0.35), y: canvas.y + Math.round(canvas.height * 0.4) };
  const to = { x: from.x + 200, y: from.y + 60 };
  let started = performance.now();
  const stroke = await live.insert(`computer:observation/${view.details.observation_id}?delivery=foreground`,
    { action: "drag", from_x: from.x, from_y: from.y, to_x: to.x, to_y: to.y, duration_ms: 200 });
  const strokeMs = Math.round(performance.now() - started);
  const strokeText = presentComputerResult(stroke);
  console.log(`stroke ${strokeMs} ms\n${strokeText}`);
  expect(stroke.details.status, strokeText).toBe("delivered");
  const pixels = stroke.details.successor!.diff!.pixels!;
  // The largest changed area is the stroke; others, such as the Undo button turning on, are listed separately.
  const [area] = pixels.areas;
  expect(area.x).toBeLessThanOrEqual(from.x);
  expect(area.y).toBeLessThanOrEqual(from.y);
  expect(area.x + area.width).toBeGreaterThanOrEqual(to.x);
  expect(area.y + area.height).toBeGreaterThanOrEqual(to.y);
  expect(area.width).toBeLessThan(260);
  expect(pixels.share).toBeLessThan(0.05);
  // LPT-928: image 1 outlines the numbered areas on the window; the stroke ends at the input point, so it is area 1.
  expect(strokeText).toMatch(/Pixels changed in \d+ areas? .*image 1 outlines and numbers them on the window:\n  1\. x=\d+ y=\d+ w=\d+ h=\d+ \(at the input point\); images 2 and 3 show .* before and after/);
  const [overview, ...crops] = stroke.content.filter(block => block.type === "image") as { data: string }[];
  expect(crops.length).toBe(2 * pixels.crops!.length);
  await mkdir(".tmp/lpt915", { recursive: true });
  await writeFile(".tmp/lpt915/stroke-overview.png", Buffer.from(overview.data, "base64"));
  await Promise.all(crops.map((crop, i) => writeFile(`.tmp/lpt915/stroke-${Math.floor(i / 2) + 1}-${i % 2 ? "after" : "before"}.png`, Buffer.from(crop.data, "base64"))));

  // The strip between the ribbon and the canvas does nothing when clicked. Background delivery leaves the real
  // pointer alone, so Paint's status bar keeps showing the same pointer position.
  const empty = { x: canvas.x + Math.round(canvas.width / 3), y: canvas.y - 12 };
  started = performance.now();
  const click = await live.insert(`computer:observation/${stroke.details.successor!.observation_id}`, { action: "click", ...empty });
  const clickMs = Math.round(performance.now() - started);
  const clickText = presentComputerResult(click);
  console.log(`empty click ${clickMs} ms\n${clickText}`);
  expect(clickText).toMatch(/Nothing changed in the UI or the window's pixels within \d+ ms after the action\./);
  expect(click.content.some(block => block.type === "image")).toBe(false);
});