Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

16.6 KB
Raw Blame
import { expect, test, vi } from "vitest";
import sharp from "sharp";
import { ComputerSession, type ComputerMode, type Driver, type DriverResult } from "../src/session.ts";
import { WebError, type WebBrowser, type WebLauncher } from "../src/web-browser.ts";
import type { WindowObserver } from "../src/native-visuals.ts";

// LPT-850: Playwright attaches to a running Chrome or Edge only in YOLO, binds it to the addressed window, and the
// session's close detaches through the worker instead of leaving the browser to anyone else.
const window = { pid: 4100, window_id: 77, title: "Attach target - Personal - Microsoft Edge", app_name: "msedge.exe", bounds: { x: 0, y: 0, width: 800, height: 600 } };

function fixture(observer?: WindowObserver) {
  let mode: ComputerMode = "yolo";
  const call = vi.fn<Driver["call"]>(async (name): Promise<DriverResult> => {
    if (name === "list_windows") return { content: [], structuredContent: { windows: [window] } };
    throw new Error(`Unexpected native call ${name}`);
  });
  const browser = { close: vi.fn(async () => {}), request: vi.fn(async () => ({})), page: vi.fn(async () => ({ url: "data:", title: "Attach target", snapshot: "- textbox \"Field\"", tabs: [], events: [] })) };
  const launch = vi.fn<WebLauncher>(async () => ({ browser: browser as unknown as WebBrowser,
    opened: { pid: window.pid, version: "155", url: "data:", title: "Attach target", via: "cdp" } }));
  const session = new ComputerSession({ call, identity: vi.fn(async () => `PiComputerUse.${"a".repeat(32)}:4100:1`), close: vi.fn(async () => {}) }, () => mode, undefined, observer, launch);
  const ref = async () => ((await session.windows()).details.state!.windows as { window_ref: string }[])[0].window_ref;
  return { session, launch, browser, call, ref, setMode(value: ComputerMode) { mode = value; } };
}

test("Playwright activity stays source-associated without exposing text or inventing a desktop point", async () => {
  const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" })); const f = fixture(observer);
  const ref = await f.ref(); await f.session.attachBrowser(ref);
  const read = await f.session.observeDom(ref);
  expect(observer.mock.calls.at(-1)?.[0]).toMatchObject({ bounds: window.bounds, mark: { kind: "inspect" } });
  observer.mockClear();
  const result = await f.session.actDom(ref, read.details.observation_id!, { action: "fill", target: { ref: "e1" }, text: "synthetic-secret" });
  expect(result.details.status).toBe("delivered");
  expect(observer).toHaveBeenCalledTimes(1);
  expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "action" }, action: "fill", outcome: "delivered" });
  expect(observer.mock.calls[0][0]).not.toHaveProperty("mark.point");
  expect(observer.mock.calls[0][0]).not.toHaveProperty("mark.region");
  expect(JSON.stringify(observer.mock.calls)).not.toContain("synthetic-secret");
  await f.session.close();
});
test("browser crop presentation uses the final derived pixels returned to the agent", async () => {
  const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" })); const f = fixture(observer);
  const ref = await f.ref(); await f.session.attachBrowser(ref);
  const png = await sharp({ create: { width: 6, height: 4, channels: 3, background: "blue" } }).png().toBuffer();
  f.browser.request.mockResolvedValue({ url: "data:", title: "Attach target", png: png.toString("base64"),
    derive: { geometry: { dpr: 1, scrollX: 0, scrollY: 0, innerWidth: 6, innerHeight: 4 }, box: { x: 2, y: 1, width: 2, height: 2 } } });
  const result = await f.session.webImage(ref, undefined, "crop", { ref: "e1" });
  expect(result.details.status).toBe("observed");
  const image = result.content.find(block => block.type === "image");
  expect(image?.type).toBe("image");
  const decoded = await sharp(Buffer.from(image!.data!, "base64")).raw().toBuffer({ resolveWithObject: true });
  expect(decoded.info).toMatchObject({ width: 2, height: 2, channels: 3 });
  expect([...decoded.data]).toEqual(Array.from({ length: 4 }, () => [0, 0, 255]).flat());
  expect(observer).toHaveBeenCalledTimes(1);
  expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "inspect" }, image });
  expect(observer.mock.calls[0][0]).not.toHaveProperty("mark.point");
  expect(observer.mock.calls[0][0]).not.toHaveProperty("mark.region");
  await f.session.close();
});
test("opening a browser selects the page window instead of an owned browser dialog", async () => {
  const f = fixture();
  f.call.mockImplementation(async name => {
    if (name === "list_windows") return { content: [], structuredContent: { windows: [
      { ...window, window_id: 78, title: "Translate page?" }, window,
    ] } };
    throw new Error(`Unexpected native call ${name}`);
  });
  const opened = await f.session.openBrowser({ browser: "msedge", profile: "agent" });
  expect(opened.details.status).toBe("observed");
  expect(opened.details.target?.window_id).toBe(window.window_id);
  await f.session.close();
});
test("Firefox Nightly opens its browser window rather than a dialog", async () => {
  const f = fixture();
  f.call.mockImplementation(async name => {
    if (name === "list_windows") return { content: [], structuredContent: { windows: [
      { ...window, window_id: 78, title: "Translate page?" },
      { ...window, title: "Attach target — Nightly", app_name: "firefox.exe" },
    ] } };
    throw new Error(`Unexpected native call ${name}`);
  });
  const opened = await f.session.openBrowser({ browser: "firefox" });
  expect(opened.details.status).toBe("observed");
  expect(opened.details.target?.window_id).toBe(window.window_id);
  await f.session.close();
});
test("opening waits when the first visible window is only a browser dialog", async () => {
  const f = fixture();
  let reads = 0;
  f.call.mockImplementation(async name => {
    if (name === "list_windows") return { content: [], structuredContent: { windows: ++reads === 1 ? [{ ...window, window_id: 78, title: "Translate page?" }] : [window] } };
    throw new Error(`Unexpected native call ${name}`);
  });
  const opened = await f.session.openBrowser({ browser: "msedge", profile: "agent" });
  expect(opened.details.target?.window_id).toBe(window.window_id);
  expect(reads).toBeGreaterThan(1);
  await f.session.close();
});
test("Safe mode only attaches to its own agent profile after approval", async () => {
  const f = fixture();
  f.setMode("safe");
  const opened = await f.session.openBrowser({ browser: "chrome", profile: "agent" });
  expect(opened.details.status).toBe("observed");
  expect(f.launch).toHaveBeenCalledWith({ browser: "chrome", profile: "agent", approval: true }, undefined);
  const ref = opened.details.window_ref!;
  expect((await f.session.observeDom(ref)).details.status).toBe("refused");
  expect((await f.session.attachBrowser(ref)).details.status).toBe("delivered");
  expect(f.browser.request).toHaveBeenCalledWith("approve", {}, undefined, expect.any(Number));
  expect((await f.session.observeDom(ref)).details.status).toBe("observed");
  await f.session.close();
  expect(f.browser.close).toHaveBeenCalledTimes(1);
});
test("a YOLO agent browser blocks Safe until it is closed", async () => {
  const f = fixture();
  expect(f.session.hasUnapprovedAgentBrowser()).toBe(false);
  await f.session.openBrowser({ browser: "chrome", profile: "agent" });
  expect(f.session.hasUnapprovedAgentBrowser()).toBe(true);
  await f.session.close();
  expect(f.session.hasUnapprovedAgentBrowser()).toBe(false);
});

test("an in-flight YOLO agent launch blocks Safe and rejects a concurrent mode change", async () => {
  const f = fixture();
  let finish!: () => void;
  const waiting = new Promise<void>(resolve => { finish = resolve; });
  f.launch.mockImplementationOnce(async () => { await waiting; return { browser: f.browser as unknown as WebBrowser, opened: { pid: window.pid, version: "", url: "", title: "" } }; });
  const opening = f.session.openBrowser({ browser: "chrome", profile: "agent" });
  await vi.waitFor(() => expect(f.launch).toHaveBeenCalled());
  expect(f.session.hasUnapprovedAgentBrowser()).toBe(true);
  f.setMode("safe");
  finish();
  expect((await opening).details.status).toBe("refused");
  expect(f.browser.close).toHaveBeenCalledOnce();
  expect(f.session.hasUnapprovedAgentBrowser()).toBe(false);
  await f.session.close();
});
test("an uncertain YOLO launch remains unknown and blocks Safe", async () => {
  const f = fixture();
  f.launch.mockRejectedValueOnce(new WebError("cleanup unconfirmed", false, true));
  const opened = await f.session.openBrowser({ browser: "chrome", profile: "agent" });
  expect(opened.details.status).toBe("unknown");
  expect(f.session.hasUnapprovedAgentBrowser()).toBe(true);
  expect((await f.session.close()).details.status).toBe("unknown");
  expect(f.session.hasUnapprovedAgentBrowser()).toBe(true);
});

test("an approved connection reports uncertain deferred navigation without losing the attachment", async () => {
  const f = fixture();
  f.setMode("safe");
  const opened = await f.session.openBrowser({ browser: "chrome", profile: "agent", url: "https://example.invalid" });
  f.browser.request.mockResolvedValueOnce({ pid: window.pid, version: "155", url: "about:blank", title: "", via: "toggle", navigation_error: "connection reset" });
  const attached = await f.session.attachBrowser(opened.details.window_ref!);
  expect(attached.details.status).toBe("unknown");
  expect(attached.content[0]).toMatchObject({ type: "text", text: expect.stringContaining("navigation outcome is unknown") });
  expect((await f.session.observeDom(opened.details.window_ref!)).details.status).toBe("observed");
  await f.session.close();
});
test("a cancelled approval reports an unknown outcome and cleanup", async () => {
  const f = fixture();
  f.setMode("safe");
  const opened = await f.session.openBrowser({ browser: "chrome", profile: "agent" });
  f.browser.request.mockRejectedValueOnce(new WebError("cancelled", false, true));
  const attached = await f.session.attachBrowser(opened.details.window_ref!);
  expect(attached.details.status).toBe("unknown");
  expect(attached.content[0]).toMatchObject({ type: "text", text: expect.stringContaining("cleanup confirmed") });
  expect(f.browser.close).toHaveBeenCalledOnce();
  await f.session.close();
});
test("a mode change stops native input to a pending Safe browser during a sequence", async () => {
  const f = fixture();
  f.setMode("safe");
  const opened = await f.session.openBrowser({ browser: "msedge", profile: "agent" });
  f.setMode("yolo");
  const png = await sharp({ create: { width: 40, height: 30, channels: 3, background: "red" } }).png().toBuffer();
  const image = { type: "image" as const, mimeType: "image/png", data: png.toString("base64") };
  let inputs = 0;
  f.call.mockImplementation(async name => {
    if (name === "list_windows") return { content: [], structuredContent: { windows: [window] } };
    if (name === "get_window_state") return { content: [image], structuredContent: { pid: window.pid, window_id: window.window_id,
      window_bounds: window.bounds, screenshot_width: 40, screenshot_height: 30, elements: [] } };
    if (name === "press_key") { inputs++; f.setMode("safe"); return { content: [], structuredContent: { effect: "confirmed" } }; }
    throw new Error(`Unexpected native call ${name}`);
  });
  const ref = opened.details.window_ref!;
  const observed = await f.session.observe(ref);
  const result = await f.session.act({ window_ref: ref, observation_id: observed.details.observation_id!, action: "sequence",
    sequence: [{ action: "key", key: "a" }, { action: "key", key: "b" }] });
  expect(result.details.status).toBe("unknown");
  expect(inputs).toBe(1);
  await f.session.close();
});
test("pending browser input in flight blocks a switch to Safe", async () => {
  const f = fixture();
  f.setMode("safe");
  const opened = await f.session.openBrowser({ browser: "msedge", profile: "agent" });
  f.setMode("yolo");
  const png = await sharp({ create: { width: 40, height: 30, channels: 3, background: "red" } }).png().toBuffer();
  const image = { type: "image" as const, mimeType: "image/png", data: png.toString("base64") };
  let release!: () => void;
  const waiting = new Promise<void>(resolve => { release = resolve; });
  f.call.mockImplementation(async name => {
    if (name === "list_windows") return { content: [], structuredContent: { windows: [window] } };
    if (name === "get_window_state") return { content: [image], structuredContent: { pid: window.pid, window_id: window.window_id,
      window_bounds: window.bounds, screenshot_width: 40, screenshot_height: 30, elements: [] } };
    if (name === "press_key") { await waiting; return { content: [], structuredContent: { effect: "confirmed" } }; }
    throw new Error(`Unexpected native call ${name}`);
  });
  const ref = opened.details.window_ref!;
  const observed = await f.session.observe(ref);
  const action = f.session.act({ window_ref: ref, observation_id: observed.details.observation_id!, action: "key", key: "a" });
  await vi.waitFor(() => expect(f.session.hasUnapprovedAgentBrowser()).toBe(true));
  release(); await action;
  expect(f.session.hasUnapprovedAgentBrowser()).toBe(false);
  await f.session.close();
});
test("input to a sibling dialog blocks Safe until it completes", async () => {
  const f = fixture();
  f.setMode("safe"); await f.session.openBrowser({ browser: "msedge", profile: "agent" });
  f.setMode("yolo");
  const dialog = { ...window, window_id: 78, title: "Allow remote debugging?" };
  const png = await sharp({ create: { width: 40, height: 30, channels: 3, background: "red" } }).png().toBuffer();
  const image = { type: "image" as const, mimeType: "image/png", data: png.toString("base64") };
  let release!: () => void;
  const waiting = new Promise<void>(resolve => { release = resolve; });
  f.call.mockImplementation(async name => {
    if (name === "list_windows") return { content: [], structuredContent: { windows: [window, dialog] } };
    if (name === "get_window_state") return { content: [image], structuredContent: { pid: dialog.pid, window_id: dialog.window_id,
      window_bounds: dialog.bounds, screenshot_width: 40, screenshot_height: 30, elements: [] } };
    if (name === "press_key") { await waiting; return { content: [], structuredContent: { effect: "confirmed" } }; }
    throw new Error(`Unexpected native call ${name}`);
  });
  const windows = await f.session.windows();
  const ref = (windows.details.state!.windows as { window_ref: string; window_id: number }[]).find(item => item.window_id === 78)!.window_ref;
  const observed = await f.session.observe(ref);
  const action = f.session.act({ window_ref: ref, observation_id: observed.details.observation_id!, action: "key", key: "a" });
  await vi.waitFor(() => expect(f.session.hasUnapprovedAgentBrowser()).toBe(true));
  release(); await action;
  expect(f.session.hasUnapprovedAgentBrowser()).toBe(false);
  await f.session.close();
});
test("Safe mode refuses before any Playwright work", async () => {
  const f = fixture();
  const ref = await f.ref();
  f.setMode("safe");
  const result = await f.session.attachBrowser(ref);
  expect(result.details.status).toBe("refused");
  expect(f.launch).not.toHaveBeenCalled();
});

test("a YOLO attach completing after Safe was selected is detached without publishing browser data", async () => {
  const f = fixture();
  const ref = await f.ref();
  let finish!: () => void;
  const waiting = new Promise<void>(resolve => { finish = resolve; });
  f.launch.mockImplementationOnce(async () => { await waiting; return { browser: f.browser as unknown as WebBrowser, opened: { pid: window.pid, version: "155", url: "https://private.invalid", title: "Private" } }; });
  const attaching = f.session.attachBrowser(ref);
  await vi.waitFor(() => expect(f.launch).toHaveBeenCalled());
  f.setMode("safe"); finish();
  const result = await attaching;
  expect(result.details.status).toBe("refused");
  expect(JSON.stringify(result)).not.toContain("private.invalid");
  expect(f.browser.close).toHaveBeenCalledOnce();
  await f.session.close();
});
test("an attached window serves DOM reads, refuses a second attach, and the session close detaches", async () => {
  const f = fixture();
  const ref = await f.ref();
  expect((await f.session.observeDom(ref)).details.status).toBe("refused");
  const attached = await f.session.attachBrowser(ref);
  expect(attached.details.status).toBe("delivered");
  // The worker receives the window's own process and title, which pick the visible tab.
  expect(f.launch).toHaveBeenCalledWith({ attach: { pid: window.pid, title: window.title } }, undefined);
  expect((await f.session.observeDom(ref)).details.status).toBe("observed");
  expect((await f.session.attachBrowser(ref)).details.status).toBe("refused");
  expect(f.launch).toHaveBeenCalledTimes(1);
  await f.session.close();
  expect(f.browser.close).toHaveBeenCalledTimes(1);
});