Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

10.8 KB
Raw Blame
import { expect, test, vi } from "vitest";
import sharp from "sharp";
import { ComputerSession, type ComputerMode, type Driver, type DriverResult } from "../src/session.ts";

const left = { display_id: "\\\\.\\DISPLAY2", monitor_handle: "0x10002", monitor_device_paths: ["native-left"],
  bounds: { x: -1000, y: -200, width: 1000, height: 800 }, work_area: { x: -1000, y: -200, width: 1000, height: 760 },
  primary: false, scale_factor: 1.5, dpi: null };
const primary = { ...left, display_id: "\\\\.\\DISPLAY1", monitor_handle: "0x10001", monitor_device_paths: ["native-primary"],
  bounds: { x: 0, y: 0, width: 1000, height: 800 }, primary: true };
async function fixture() {
  let mode: ComputerMode = "yolo"; let topology = "native-topology";
  const raw = await sharp({ create: { width: 1000, height: 800, channels: 3, background: "blue" } }).png().toBuffer();
  const small = await sharp({ create: { width: 500, height: 400, channels: 3, background: "red" } }).png().toBuffer();
  const driver = {
    selectedDisplays: true,
    call: vi.fn<Driver["call"]>(async (name, args): Promise<DriverResult> => {
      if (name === "list_displays") return { content: [], structuredContent: { platform: "windows", topology_id: topology, displays: [primary, left] } };
      if (name === "get_display_state") {
        if (args.topology_id !== topology) return { content: [], isError: true, structuredContent: { status: "refused", code: "display_topology_changed" } };
        const full = args.max_image_dimension === 0;
        return { content: [{ type: "image", mimeType: "image/png", data: (full ? raw : small).toString("base64") }],
          structuredContent: { platform: "windows", topology_id: topology, display: { ...(args.display_id === primary.display_id ? primary : left) },
            screenshot_width: full ? 1000 : 500, screenshot_height: full ? 800 : 400, screenshot_original_width: 1000, screenshot_original_height: 800 } };
      }
      throw new Error(`Unexpected native fallback ${name}`);
    }), identity: vi.fn(async () => "unused"), close: vi.fn(async () => {}),
  };
  const session = new ComputerSession(driver, () => mode);
  return { driver, session, setMode(value: ComputerMode) { mode = value; }, changeTopology() { topology = "changed"; } };
}
async function discover(f: Awaited<ReturnType<typeof fixture>>) {
  const result = await f.session.displays();
  expect(result.details.status).toBe("listed");
  const displays = result.details.state!.displays as { display_ref: string; display_id: string }[];
  return { result, ref: displays.find(d => d.display_id === left.display_id)!.display_ref };
}

test("native discovery and selected pixels keep monitor identity, negative origin and image-local geometry separate", async () => {
  const f = await fixture(); const { result: listed, ref } = await discover(f);
  expect(listed.details.state!.displays).toHaveLength(2);
  const overview = await f.session.observeDisplay(ref);
  expect(overview.details.status).toBe("observed");
  expect(f.driver.call).toHaveBeenLastCalledWith("get_display_state", { display_id: left.display_id, topology_id: "native-topology", max_image_dimension: 1280 }, undefined);
  expect(overview.details.source).toMatchObject({ identity: { display_ref: ref, display_id: left.display_id }, membership: { monitor_id: left.display_id },
    image: { scope: "selected_display", origin: { x: 0, y: 0, relative_to: "display" }, native_bounds: left.bounds, coordinate_space: "display_image_pixels" },
    scale: { native_scale_factor: 1.5, dpi: null }, capabilities: { input: "none" } });
  expect(overview.details).not.toHaveProperty("observation_id");
  const count = f.driver.call.mock.calls.length;
  expect((await f.session.act({ window_ref: ref, observation_id: overview.details.agent_image!.image_id as string, action: "click", x: 1, y: 1 })).details.status).toBe("refused");
  expect(f.driver.call).toHaveBeenCalledTimes(count);
});

test("overview metadata follows the trusted runtime filter while fresh display detail stays Lanczos3", async () => {
  const f = await fixture();
  Object.assign(f.driver, { overviewKernel: "bilinear" });
  const { ref } = await discover(f);
  const overview = await f.session.observeDisplay(ref);
  expect(overview.details.agent_image).toMatchObject({ configuration: { kernel: "bilinear" } });
  const detail = await f.session.observeDisplay(ref, undefined, {
    image_id: overview.details.agent_image!.image_id as string, x: 100, y: 80, width: 100, height: 80,
  });
  expect(detail.details.status).toBe("observed");
  expect(detail.details.agent_image).toMatchObject({ configuration: { kernel: "lanczos3" } });
});

test("selected detail uses fresh selected-monitor pixels and never a primary screenshot or host-global crop", async () => {
  const f = await fixture(); const { ref } = await discover(f); const overview = await f.session.observeDisplay(ref);
  const crop = await f.session.observeDisplay(ref, undefined, { image_id: overview.details.agent_image!.image_id as string, x: 100, y: 80, width: 100, height: 80 });
  expect(crop.details.status).toBe("observed");
  const image = crop.content.find(b => b.type === "image")!; if (image.type !== "image") throw Error("missing crop");
  const pixel = await sharp(Buffer.from(image.data, "base64")).raw().toBuffer();
  expect([...pixel.subarray(0, 3)]).toEqual([0, 0, 255]); // Fresh blue native pixels, not the red overview.
  expect(crop.details.source).toMatchObject({ image: { scope: "selected_display_region", origin: { x: 200, y: 160, relative_to: "display" }, native_bounds: left.bounds }, capabilities: { input: "none", detail: "unsupported" } });
  expect(crop.details.agent_image).toMatchObject({ region: { native: { left: 200, top: 160, width: 200, height: 160 } } });
  expect(f.driver.call).toHaveBeenLastCalledWith("get_display_state", { display_id: left.display_id, topology_id: "native-topology", max_image_dimension: 0 }, undefined);
});

test("changed topology refuses a crop without output pixels or primary fallback", async () => {
  const f = await fixture(); const { ref } = await discover(f); const overview = await f.session.observeDisplay(ref); f.changeTopology();
  const result = await f.session.observeDisplay(ref, undefined, { image_id: overview.details.agent_image!.image_id as string, x: 1, y: 1, width: 10, height: 10 });
  expect(result.details.status).toBe("refused"); expect(result.content.every(b => b.type === "text")).toBe(true);
  expect(f.driver.call.mock.calls.every(([name]) => name === "list_displays" || name === "get_display_state")).toBe(true);
});

test("Safe and unsupported runtimes cannot enumerate or dispatch selected-display reads", async () => {
  const f = await fixture(); f.setMode("safe"); expect((await f.session.displays()).details.status).toBe("refused");
  expect(f.driver.call).not.toHaveBeenCalled(); f.setMode("yolo"); f.driver.selectedDisplays = false;
  expect((await f.session.displays()).details.status).toBe("refused"); expect((await f.session.observeDisplay("missing-ref")).details.status).toBe("refused");
  expect(f.driver.call).not.toHaveBeenCalled();
});

test("revocation during discovery/capture withholds native metadata and pixels", async () => {
  for (const phase of ["list_displays", "get_display_state"]) {
    const f = await fixture(); let ref: string | undefined;
    if (phase === "get_display_state") ref = (await discover(f)).ref;
    const call = f.driver.call; f.driver.call = vi.fn(async (...args) => { const result = await call(...args); f.setMode("safe"); return result; });
    const result = ref ? await f.session.observeDisplay(ref) : await f.session.displays();
    expect(result.details.status).toBe("refused"); expect(result.content.every(b => b.type === "text")).toBe(true);
    expect(result.details).not.toHaveProperty("state"); expect(result.details).not.toHaveProperty("source");
  }
});

test("each monitor keeps its own latest overview and a crop cannot cross source references", async () => {
  const f = await fixture(); const { result, ref } = await discover(f);
  const primaryRef = (result.details.state!.displays as { display_ref: string; display_id: string }[]).find(d => d.display_id === primary.display_id)!.display_ref;
  const leftView = await f.session.observeDisplay(ref); const primaryView = await f.session.observeDisplay(primaryRef);
  const request = { image_id: leftView.details.agent_image!.image_id as string, x: 1, y: 1, width: 10, height: 10 };
  const count = f.driver.call.mock.calls.length;
  expect((await f.session.observeDisplay(primaryRef, undefined, request)).details.status).toBe("refused");
  expect(f.driver.call).toHaveBeenCalledTimes(count);
  expect((await f.session.observeDisplay(ref, undefined, request)).details.status).toBe("observed");
  expect((await f.session.observeDisplay(ref, undefined, request)).details.status).toBe("refused");
  expect((await f.session.observeDisplay(primaryRef, undefined, { ...request, image_id: primaryView.details.agent_image!.image_id as string })).details.status).toBe("observed");
});

test("rediscovery keeps same-topology refs but retires changed topology and close retires all refs", async () => {
  const f = await fixture(); const { ref } = await discover(f);
  expect((await discover(f)).ref).toBe(ref); f.changeTopology();
  const next = (await discover(f)).ref; expect(next).not.toBe(ref);
  const count = f.driver.call.mock.calls.length;
  expect((await f.session.observeDisplay(ref)).details.status).toBe("refused"); expect(f.driver.call).toHaveBeenCalledTimes(count);
  await f.session.close(); expect((await f.session.observeDisplay(next)).details.status).toBe("refused");
});

test("cancellation during native selected capture withholds pixels and does not leave a crop grant", async () => {
  const f = await fixture(); const { ref } = await discover(f); const abort = new AbortController();
  const call = f.driver.call; f.driver.call = vi.fn(async (...args) => { const result = await call(...args); abort.abort(); return result; });
  const result = await f.session.observeDisplay(ref, abort.signal);
  expect(result.details.status).toBe("refused"); expect(result.content.every(b => b.type === "text")).toBe(true);
  expect(result.details).not.toHaveProperty("agent_image");
});

test("selected-monitor observations carry the exact physical monitor source and returned image", async () => {
  const f = await fixture(); const observer = vi.fn(async () => ({ status: "shown" as const }));
  const session = new ComputerSession(f.driver, undefined, undefined, observer);
  const listed = await session.displays();
  const ref = (listed.details.state!.displays as { display_ref: string }[])[1].display_ref;
  const view = await session.observeDisplay(ref);
  expect(view.details.status).toBe("observed");
  const display = left;
  expect(observer).toHaveBeenCalledWith({ display: display.display_id, bounds: display.bounds, image: view.content.find(block => block.type === "image") }, undefined);
  expect(view.details.highlight).toMatchObject({ status: "shown" });
});