Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

14.2 KB
Raw Blame
import { expect, test, vi } from "vitest";
import sharp from "sharp";
import * as detailProcessing from "../src/native-detail.ts";
import { ComputerSession, type ComputerMode, type Driver, type DriverResult } from "../src/session.ts";

async function fixture() {
  let mode: ComputerMode = "yolo";
  let bounds = { x: 10, y: 20, width: 1000, height: 800 };
  const raw = await sharp({ create: { width: 1000, height: 800, channels: 3, background: "red" } })
    .composite([{ input: await sharp({ create: { width: 200, height: 160, channels: 3, background: "blue" } }).png().toBuffer(), left: 200, top: 160 }]).png().toBuffer();
  const small = await sharp(raw).resize(500, 400).png().toBuffer();
  const jpeg = await sharp(raw).extract({ left: 100, top: 80, width: 280, height: 240 }).jpeg().toBuffer();
  const png = (bytes: Buffer) => ({ type: "image" as const, mimeType: "image/png", data: bytes.toString("base64") });
  const driver: Driver = {
    call: vi.fn<Driver["call"]>(async (name, args): Promise<DriverResult> => {
      if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid: 42, window_id: 91, bounds }] } };
      if (name === "get_window_state") return { content: [png(small)], structuredContent: {
        pid: 42, window_id: 91, screenshot_width: 500, screenshot_height: 400, window_bounds: { ...bounds }, capture_id: "window-capture",
        elements: [{ element_token: "button", label: "test", actions: ["click"] }],
      } };
      if (name === "zoom") return { content: [{ type: "image", mimeType: "image/jpeg", data: jpeg.toString("base64") }],
        structuredContent: { width: 280, height: 240, x_offset: 100, y_offset: 80 } };
      if (name === "get_desktop_state") { const native = args.max_image_dimension === 0;
        return { content: [png(native ? raw : small)], structuredContent: { display: "primary", platform: "windows", capture_id: "private-display-id",
          screenshot_width: native ? 1000 : 500, screenshot_height: native ? 800 : 400, screen_width: 1000, screen_height: 800,
          screenshot_original_width: 1000, screenshot_original_height: 800, scale_factor: 1 } }; }
      throw new Error(`Unexpected native call ${name}`);
    }), identity: vi.fn(async () => "lifetime"), close: vi.fn(async () => {}),
  };
  const session = new ComputerSession(driver, () => mode);
  return { session, driver, raw, setMode(value: ComputerMode) { mode = value; }, setBounds(value: typeof bounds) { bounds = value; } };
}
function imageId(view: Awaited<ReturnType<ComputerSession["observeDisplay"]>>) { return view.details.agent_image?.image_id as string; }
const rect = { x: 100, y: 80, width: 100, height: 80 };

test("runtime reset retires earlier display geometry even if the next request restarts its peer", async () => {
  const f = await fixture(); const view = await f.session.observeDisplay("primary"); const call = f.driver.call;
  f.driver.call = vi.fn<Driver["call"]>(async (...args) => {
    if (args[0] === "list_apps") throw Error("peer disconnected");
    return call(...args);
  });
  expect((await f.session.apps()).details.status).toBe("refused");
  const count = vi.mocked(f.driver.call).mock.calls.length;
  expect((await f.session.observeDisplay("primary", undefined, { ...rect, image_id: imageId(view) })).details.status).toBe("refused");
  expect(f.driver.call).toHaveBeenCalledTimes(count);
});

test("an overview whose raster and reported dimensions disagree cannot ground a detail", async () => {
  const f = await fixture(); const call = f.driver.call;
  f.driver.call = vi.fn<Driver["call"]>(async (...args) => {
    const result = await call(...args);
    if (args[0] === "get_desktop_state") result.structuredContent!.screenshot_width = 499;
    return result;
  });
  const view = await f.session.observeDisplay("primary"); const count = vi.mocked(f.driver.call).mock.calls.length;
  expect((await f.session.observeDisplay("primary", undefined, { ...rect, image_id: imageId(view) })).details.status).toBe("refused");
  expect(f.driver.call).toHaveBeenCalledTimes(count);
});

test("revocation and cancellation during adapter processing withhold final pixels", async () => {
  for (const cancel of [false, true]) {
    const f = await fixture(); const overview = await f.session.observeDisplay("primary"); const controller = new AbortController();
    const crop = detailProcessing.cropDisplay;
    const processing = vi.spyOn(detailProcessing, "cropDisplay").mockImplementation(async (...args) => {
      const result = await crop(...args); if (cancel) controller.abort(); else f.setMode("safe"); return result;
    });
    try {
      const result = await f.session.observeDisplay("primary", controller.signal, { ...rect, image_id: imageId(overview) });
      expect(result.details.status).toBe("refused"); expect(result.content.every(b => b.type === "text")).toBe(true);
      expect(result.details).not.toHaveProperty("source"); expect(JSON.stringify(result)).not.toContain("private-display-id");
    } finally { processing.mockRestore(); }
  }
});

test("detail scaling is bounded and fractional rectangles map outwards once", async () => {
  const f = await fixture(); const overview = await f.session.observeDisplay("primary");
  const result = await f.session.observeDisplay("primary", undefined, { image_id: imageId(overview), x: 100.25, y: 80.25, width: 99.5, height: 79.5, maxLongEdge: 100, scale: 0.5 });
  expect(result.details.status).toBe("observed");
  expect(result.details.agent_image).toMatchObject({ region: { native: { left: 200, top: 160, width: 200, height: 160 } }, output: { width: 50, height: 40 } });
});

test("Safe selected or native handle lost during window zoom withholds its pixels", async () => {
  for (const revoke of [false, true]) {
    const f = await fixture(); const discovered = await f.session.windows();
    const ref = (discovered.details.state!.windows as {window_ref:string}[])[0].window_ref;
    const overview = await f.session.observe(ref); const call = f.driver.call;
    f.driver.call = vi.fn<Driver["call"]>(async (...args) => {
      const result = await call(...args);
      if(args[0] === "zoom") { if(revoke) f.setMode("safe"); else vi.mocked(f.driver.identity).mockResolvedValue(undefined); }
      return result;
    });
    const result = await f.session.observe(ref, undefined, undefined, { ...rect, image_id: imageId(overview) });
    expect(result.details.status).toBe("refused"); expect(result.content.every(b => b.type === "text")).toBe(true);
  }
});

test("display detail extracts fresh native pixels, reports mapping and never enlarges or publishes the raw display", async () => {
  const f = await fixture(); const overview = await f.session.observeDisplay("primary");
  expect(imageId(overview)).toEqual(expect.any(String));
  const detail = await f.session.observeDisplay("primary", undefined, { ...rect, image_id: imageId(overview), maxLongEdge: 1200, scale: 1 });
  expect(detail.details.status).toBe("observed");
  expect(f.driver.call).toHaveBeenLastCalledWith("get_desktop_state", { max_image_dimension: 0 }, undefined);
  expect(detail.details.source).toMatchObject({ kind: "display", image: { scope: "primary_display_region", coordinate_space: "detail_image_pixels",
    dimensions: { width: 200, height: 160 }, origin: { x: 200, y: 160, relative_to: "primary_display" } }, capabilities: { input: "none" } });
  const image = detail.content.find(block => block.type === "image")!;
  if (image.type !== "image") throw Error("Missing detail");
  const pixel = await sharp(Buffer.from(image.data, "base64")).removeAlpha().raw().toBuffer();
  expect([...pixel.subarray(0, 3)]).toEqual([0, 0, 255]);
  expect(JSON.stringify(detail)).not.toContain(f.raw.toString("base64"));
  expect(JSON.stringify(detail)).not.toContain("private-display-id");
  expect(detail.details.agent_image).toMatchObject({ route: "cua_fresh_display_crop", configuration: { maxLongEdge: 1200, scale: 1 }, region: { native: { left: 200, top: 160, width: 200, height: 160 } } });
  expect((await f.session.observeDisplay("primary", undefined, { ...rect, image_id: imageId(overview) })).details.status).toBe("refused");
  const again = await f.session.observeDisplay("primary"); expect(again.details.agent_image).toMatchObject({ configuration: { maxLongEdge: 1280, scale: 1 } });
});

test("invalid, missing and retired display regions are rejected before native work", async () => {
  const f = await fixture();
  expect((await f.session.observeDisplay("primary", undefined, { ...rect, image_id: "unknown" })).details.status).toBe("refused");
  expect(f.driver.call).not.toHaveBeenCalled();
  const first = await f.session.observeDisplay("primary"); const latest = await f.session.observeDisplay("primary"); const count = vi.mocked(f.driver.call).mock.calls.length;
  for (const zoom of [{ ...rect, image_id: imageId(first) }, { ...rect, x: -1, image_id: imageId(latest) },
    { ...rect, width: 900, image_id: imageId(latest) }, { ...rect, maxLongEdge: 9000, image_id: imageId(latest) }, { ...rect, x: NaN, image_id: imageId(latest) }])
    expect((await f.session.observeDisplay("primary", undefined, zoom)).details.status).toBe("refused");
  expect(f.driver.call).toHaveBeenCalledTimes(count);
});

test("display geometry drift, Safe revocation and cancelled capture withhold pixels and private diagnostics", async () => {
  for (const route of ["geometry", "revoke", "cancel", "private-error"]) {
    const f = await fixture(); const overview = await f.session.observeDisplay("primary"); const call = f.driver.call; const controller = new AbortController();
    f.driver.call = vi.fn<Driver["call"]>(async (...args) => {
      if (route === "private-error") { f.setMode("safe"); throw Error("PRIVATE_RAW_PIXELS"); }
      const result = await call(...args);
      if (route === "geometry") result.structuredContent!.screen_width = 1200;
      if (route === "revoke") f.setMode("safe");
      if (route === "cancel") controller.abort();
      return result;
    });
    const result = await f.session.observeDisplay("primary", controller.signal, { ...rect, image_id: imageId(overview) });
    expect(result.details.status).toBe("refused"); expect(result.content.every(b => b.type === "text")).toBe(true);
    expect(JSON.stringify(result)).not.toMatch(/PRIVATE_RAW_PIXELS|private-display-id/);
  }
  const f = await fixture(); const overview = await f.session.observeDisplay("primary"); f.setMode("safe");
  const count = vi.mocked(f.driver.call).mock.calls.length;
  expect((await f.session.observeDisplay("primary", undefined, { ...rect, image_id: imageId(overview) })).details.status).toBe("refused");
  expect(f.driver.call).toHaveBeenCalledTimes(count);
});

test("native window zoom consumes overview input and preserves CUA JPEG padding without wider capture or raise", async () => {
  const f = await fixture(); const discovered = await f.session.windows(); const ref = (discovered.details.state!.windows as { window_ref: string }[])[0].window_ref;
  const overview = await f.session.observe(ref);
  const zoom = await f.session.observe(ref, undefined, undefined, { ...rect, image_id: imageId(overview) });
  expect(zoom.details.status).toBe("observed");
  expect(f.driver.call).toHaveBeenCalledWith("zoom", { pid: 42, window_id: 91, x1: 100, y1: 80, x2: 200, y2: 160 }, undefined);
  expect(zoom.content.some(b => b.type === "image" && b.mimeType === "image/jpeg")).toBe(true);
  expect(zoom.details.source).toMatchObject({ capabilities: { input: "none" }, image: { scope: "native_window_zoom", coordinate_space: "zoom_image_pixels" } });
  expect(zoom.details).not.toHaveProperty("observation_id");
  const count = vi.mocked(f.driver.call).mock.calls.length;
  expect((await f.session.act({ window_ref: ref, observation_id: overview.details.observation_id!, action: "click", x: 10, y: 20 })).details.status).toBe("refused");
  expect((await f.session.observe(ref, undefined, undefined, { ...rect, image_id: imageId(overview) })).details.status).toBe("refused");
  expect(f.driver.call).toHaveBeenCalledTimes(count);
  expect(vi.mocked(f.driver.call).mock.calls.some(([name]) => ["bring_to_front", "get_desktop_state", "click"].includes(name))).toBe(false);
});

test("window movement, shrinking and growth preserve the original local zoom rectangle", async () => {
  for (const bounds of [{ x: 200, y: 300, width: 1000, height: 800 },
    { x: 200, y: 300, width: 300, height: 240 }, { x: 200, y: 300, width: 2000, height: 1600 }]) {
    const f = await fixture(); const listed = await f.session.windows();
    const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
    const view = await f.session.observe(ref); f.setBounds(bounds);
    const result = await f.session.observe(ref, undefined, undefined, { ...rect, image_id: imageId(view) });
    expect(result.details.status).toBe("observed");
    expect(f.driver.call).toHaveBeenCalledWith("zoom", { pid: 42, window_id: 91, x1: 100, y1: 80, x2: 200, y2: 160 }, undefined);
    expect(vi.mocked(f.driver.call).mock.calls.filter(([name]) => name === "get_window_state")).toHaveLength(1);
  }
});

test("movement during native zoom does not discard its fresh pixels", async () => {
  const f = await fixture(); const listed = await f.session.windows();
  const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
  const view = await f.session.observe(ref); const call = f.driver.call;
  f.driver.call = vi.fn<Driver["call"]>(async (...args) => {
    const result = await call(...args);
    if (args[0] === "zoom") f.setBounds({ x: 500, y: 600, width: 1100, height: 900 });
    return result;
  });
  expect((await f.session.observe(ref, undefined, undefined, { ...rect, image_id: imageId(view) })).details.status).toBe("observed");
});

test("window lifetime loss and Safe still withhold detail", async () => {
  for (const route of ["closed", "safe"]) {
    const f = await fixture(); const listed = await f.session.windows(); const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
    const view = await f.session.observe(ref);
    if (route === "closed") vi.mocked(f.driver.identity).mockResolvedValue(undefined);
    if (route === "safe") f.setMode("safe");
    expect((await f.session.observe(ref, undefined, undefined, { ...rect, image_id: imageId(view) })).details.status).toBe("refused");
    expect(vi.mocked(f.driver.call).mock.calls.some(([name]) => name === "zoom")).toBe(false);
  }
});