Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

16.3 KB
Raw Blame
import { expect, test, vi } from "vitest";
import { ComputerSession, type Driver, type WindowTarget } from "../src/session.ts";

const target = { pid: 42, window_id: 100 };
function fixture() {
  let windows = [target];
  let generation = "first-lifetime";
  let snapshot = 0;
  const driver: Driver = {
    identity: vi.fn(async (window) => `${window.pid}:${window.window_id}:${generation}`),
    call: vi.fn<Driver["call"]>(async (name, args) => {
      if (name === "list_windows") return { content: [], structuredContent: { windows: args.pid ? windows.filter(window => window.pid === args.pid) : windows } };
      if (name === "list_apps") return { content: [], structuredContent: {
        apps: [{ name: "Notepad", launch_path: "shell:appsFolder\\Microsoft.WindowsNotepad_8wekyb3d8bbwe!App", kind: "uwp" }],
      } };
      if (name === "launch_app") {
        const created = { pid: 42, window_id: 101, app_name: "Notepad.exe" };
        windows = [...windows, created];
        return { content: [{ type: "text", text: JSON.stringify({ pid: 43, windows: [created] }) }] };
      }
      if (name === "get_window_state") return {
        content: [{ type: "image", mimeType: "image/png", data: "cG5n" }],
        structuredContent: { ...args, snapshot_id: "s" + ++snapshot,
          elements: [{ element_token: `s${snapshot}:0`, role: "edit", value: "" }] },
      };
      return { content: [], structuredContent: { effect: "unverifiable" } };
    }),
    close: vi.fn(async () => {}),
  };
  return { driver, session: new ComputerSession(driver),
    replaceWindows(value: WindowTarget[]) { windows = value; },
    reuseHandle() { generation = "second-lifetime"; } };
}
async function readExisting(session: ComputerSession) {
  const found = await session.windows();
  const windows = found.details.state!.windows as { window_ref: string }[];
  const ref = windows[0].window_ref;
  return { ref, observed: await session.observe(ref) };
}

test("window observations use the shared cap and keep current host geometry separate from unknown DPI", async () => {
  const f = fixture(); const native = f.driver.call;
  let bounds = { x: 80, y: 90, width: 1000, height: 700 };
  f.driver.call = vi.fn<Driver["call"]>(async (...args) => {
    const result = await native(...args);
    return args[0] === "get_window_state" ? { ...result, structuredContent: { ...result.structuredContent,
      screenshot_width: 500, screenshot_height: 350, window_bounds: bounds } } : result;
  });
  const { ref, observed } = await readExisting(f.session);
  expect(observed.details.source).toMatchObject({ kind: "window", domain: "windows-host", identity: { window_ref: ref, ...target },
    image: { native_bounds: bounds, dimensions: { width: 500, height: 350 }, original_dimensions: null },
    scale: { native_scale_factor: null, dpi: null, device_pixel_ratio: null } });
  expect(observed.content).toContainEqual({ type: "image", mimeType: "image/png", data: "cG5n" });
  expect(vi.mocked(f.driver.call).mock.calls.find(([name]) => name === "get_window_state")?.[1]).toMatchObject({ max_image_dimension: 1280 });
  bounds = { x: 200, y: 300, width: 1000, height: 700 };
  expect((await f.session.observe(ref)).details.source?.image.native_bounds).toEqual(bounds);
});
test("discovery, observation and action share identity; native delivery evidence stays action-scoped", async () => {
  const f = fixture(); const found = await f.session.windows();
  const window = (found.details.state!.windows as { window_ref: string; source_identity: unknown }[])[0];
  const observed = await f.session.observe(window.window_ref);
  expect(observed.details.source_identity).toEqual(window.source_identity);
  expect(observed.details.source).toMatchObject(window.source_identity!);
  vi.mocked(f.driver.call).mockResolvedValueOnce({ content: [], structuredContent: {
    effect: "confirmed", route: "accessibility", delivery: { mode: "background" } } });
  const delivered = await f.session.act({ window_ref: window.window_ref, observation_id: observed.details.observation_id!,
    action: "click", element_token: "s1:0" });
  expect(delivered.details.source_identity).toEqual(window.source_identity);
  expect(delivered.details.action_capability).toMatchObject({ operation: "click", background: "delivered", foreground: "unknown" });
  const next = await f.session.observe(window.window_ref);
  vi.mocked(f.driver.call).mockResolvedValueOnce({ isError: true, content: [], structuredContent: {
    code: "background_uipi_blocked", event_kind: "keystroke", escalation: { recommended: "foreground" } } });
  const refused = await f.session.act({ window_ref: window.window_ref, observation_id: next.details.observation_id!,
    action: "key", key: "return" });
  expect(refused.details.status).toBe("refused");
  expect(refused.details.action_capability).toMatchObject({ background: "unavailable", foreground: "suggested" });
  expect((await f.session.act({ window_ref: window.window_ref, observation_id: next.details.observation_id!,
    action: "key", key: "return" })).details.status).toBe("refused");
});

test("another agent's source and observation cannot authorize native input", async () => {
  const one = fixture(), two = fixture();
  const a = await readExisting(one.session), b = await readExisting(two.session);
  expect(a.observed.details.source!.identity.session_id).not.toBe(b.observed.details.source!.identity.session_id);
  const before = vi.mocked(two.driver.call).mock.calls.length;
  for (const window_ref of [a.ref, b.ref]) {
    expect((await two.session.act({ window_ref, observation_id: a.observed.details.observation_id!,
      action: "key", key: "return" })).details.status).toBe("refused");
  }
  expect(two.driver.call).toHaveBeenCalledTimes(before);
});

test("batch receipts retain separate capability evidence for each attempted action", async () => {
  const f = fixture(); const { ref, observed } = await readExisting(f.session);
  vi.mocked(f.driver.call).mockResolvedValueOnce({ content: [], structuredContent: {
    effect: "confirmed", route: "accessibility", delivery: { mode: "background" } } })
    .mockResolvedValueOnce({ content: [], isError: true, structuredContent: {
      code: "background_unavailable", escalation: { recommended: "foreground" } } });
  const result = await f.session.act({ window_ref: ref, observation_id: observed.details.observation_id!,
    action: "sequence", sequence: [{ action: "type_text", element_token: "s1:0", text: "Disposable" }, { action: "key", key: "return" }] });
  const steps = (result.details.state!.sequence as { steps: { action_capability: unknown }[] }).steps;
  expect(steps).toHaveLength(2);
  expect(steps[0].action_capability).toMatchObject({ background: "delivered", foreground: "unknown" });
  expect(steps[1].action_capability).toMatchObject({ background: "unavailable", foreground: "suggested" });
  expect(result.details.action_capability).toBeUndefined();
});
test("a sequence spends the single-use native capture on its first pixel click only", async () => {
  const f = fixture(); const native = f.driver.call;
  f.driver.call = vi.fn<Driver["call"]>(async (...args) => {
    const result = await native(...args);
    return args[0] === "get_window_state" ? { ...result, structuredContent: { ...result.structuredContent,
      capture_id: "capture-1", screenshot_width: 500, screenshot_height: 350 } } : result;
  });
  const { ref, observed } = await readExisting(f.session);
  await f.session.act({ window_ref: ref, observation_id: observed.details.observation_id!, delivery_mode: "foreground",
    action: "sequence", sequence: [{ action: "click", x: 10, y: 10 }, { action: "click", x: 20, y: 20 }] });
  const clicks = vi.mocked(f.driver.call).mock.calls.filter(([name]) => name === "click").map(([, args]) => args.capture_id);
  expect(clicks).toEqual(["capture-1", undefined]);
});
test("starting an executable by path with arguments is YOLO-only and launches nothing in Safe", async () => {
  const f = fixture();
  const safe = new ComputerSession(f.driver, () => "safe");
  const refused = await safe.open({ path: "C:\\Tools\\app.exe", args: ["--python", "x.py"] });
  expect(refused.details.status).toBe("refused");
  expect(vi.mocked(f.driver.call).mock.calls.some(([name]) => name === "launch_app")).toBe(false);
  await f.session.open({ path: "C:\\Tools\\app.exe", args: ["--python", "x.py"] });
  expect(f.driver.call).toHaveBeenCalledWith("launch_app", { path: "C:\\Tools\\app.exe", additional_arguments: ["--python", "x.py"] }, undefined);
});
test("opens a genuinely new window and returns its first observation in the same result", async () => {
  const { session, driver } = fixture();
  const opened = await session.open("Notepad");
  expect(opened).toMatchObject({ details: { status: "observed", window_ref: expect.any(String),
    opened: true, observation_id: expect.any(String), target: { pid: 42, window_id: 101 } } });
  expect(opened.content).toContainEqual({ type: "image", mimeType: "image/png", data: "cG5n" });
  expect(driver.call).toHaveBeenCalledWith("launch_app", { aumid: "Microsoft.WindowsNotepad_8wekyb3d8bbwe!App" }, undefined);
});

test("two catalog paths with the same app name are ambiguous and do not dispatch launch", async () => {
  const { session, driver } = fixture();
  vi.mocked(driver.call).mockResolvedValueOnce({ content: [], structuredContent: { apps: [
    { name: "Notepad", launch_path: "C:\\One\\notepad.exe" },
    { name: "Notepad", launch_path: "C:\\Two\\notepad.exe" },
  ] } });
  expect((await session.open("Notepad")).details.status).toBe("refused");
  expect(driver.call).toHaveBeenCalledTimes(1);
});

test("an interrupted launch is unknown and is never relaunched automatically", async () => {
  const { session, driver } = fixture();
  const call = driver.call;
  driver.call = vi.fn(async (name, args, signal) => {
    if (name === "launch_app") throw new Error("Transport lost after launch");
    return call(name, args, signal);
  });
  const result = await session.open("Notepad");
  expect(result.details.status).toBe("unknown");
  expect(driver.close).toHaveBeenCalledOnce();
  expect(vi.mocked(driver.call).mock.calls.filter(([name]) => name === "launch_app")).toHaveLength(1);
});

test("cancellation before opening does not call native discovery or launch", async () => {
  const { session, driver } = fixture();
  const controller = new AbortController(); controller.abort();
  expect((await session.open("Notepad", controller.signal)).details.status).toBe("refused");
  expect(driver.call).not.toHaveBeenCalled();
});

test("native refusals and partial errors keep their explanation and consume the observation", async () => {
  for (const nativeState of [{ code: "stale_element_token" }, { effect: "partial" }]) {
    const { session, driver } = fixture();
    const { ref, observed } = await readExisting(session);
    vi.mocked(driver.call).mockResolvedValueOnce({ isError: true, structuredContent: nativeState,
      content: [{ type: "text", text: "Native diagnostic" }] });
    const args = { window_ref: ref, observation_id: observed.details.observation_id!,
      action: "type_text" as const, element_token: "s1:0", text: "Hello" };
    const result = await session.act(args);
    expect(result.details.status).toBe("code" in nativeState ? "refused" : "unknown");
    expect(result.details.state).toMatchObject({ ...nativeState, native_message: "Native diagnostic" });
    expect((await session.act(args)).details.status).toBe("refused");
  }
});

test("pixel input needs current scoped image geometry and cannot fall outside it", async () => {
  const { session, driver } = fixture();
  const { ref, observed } = await readExisting(session);
  const before = vi.mocked(driver.call).mock.calls.length;
  expect((await session.act({ window_ref: ref, observation_id: observed.details.observation_id!,
    action: "click", x: 9000, y: 9000 })).details.status).toBe("refused");
  expect(driver.call).toHaveBeenCalledTimes(before);
});
test("missing or ambiguous app names do not launch anything", async () => {
  const { session, driver } = fixture();
  expect((await session.open("Missing app")).details.status).toBe("refused");
  expect(vi.mocked(driver.call).mock.calls.map(([name]) => name)).not.toContain("launch_app");
});

test("a launch returning an existing window is not reported as a new one", async () => {
  const { session, driver } = fixture();
  const call = driver.call;
  driver.call = vi.fn(async (name, args, signal) => name === "launch_app"
    ? { content: [], structuredContent: { pid: 42, windows: [target] } }
    : call(name, args, signal));
  const opened = await session.open("Notepad");
  expect(opened.details.status).toBe("unknown");
  expect(opened.details).not.toHaveProperty("opened", true);
  expect(vi.mocked(driver.call).mock.calls.filter(([n]) => n === "launch_app")).toHaveLength(1);
});

test("YOLO reads existing windows, keeps separate observations, and refuses cross-window tokens", async () => {
  const { session, driver, replaceWindows } = fixture();
  replaceWindows([target, { pid: 42, window_id: 101 }]);
  const found = await session.windows();
  const refs = (found.details.state!.windows as { window_ref: string }[]).map(w => w.window_ref);
  const first = await session.observe(refs[0]);
  const second = await session.observe(refs[1]);
  expect(first.details.window_ref).not.toBe(second.details.window_ref);
  const calls = vi.mocked(driver.call).mock.calls.length;
  const wrong = await session.act({ window_ref: refs[1], observation_id: first.details.observation_id!,
    action: "type_text", text: "No", element_token: "s1:0" });
  expect(wrong.details.status).toBe("refused");
  expect(driver.call).toHaveBeenCalledTimes(calls);
  const accepted = await session.act({ window_ref: refs[0], observation_id: first.details.observation_id!,
    action: "type_text", text: "Hello", element_token: "s1:0" });
  expect(accepted.details.status).toBe("delivered");
  expect(accepted.content[0]).toMatchObject({ text: expect.stringContaining("Observe again") });
  expect(driver.call).toHaveBeenLastCalledWith("type_text", {
    target: { kind: "window", ...target }, element_token: "s1:0", text: "Hello", delivery_mode: "background",
  }, undefined);
});

test("a closed or recycled HWND retires its reference before observation or input", async () => {
  const { session, driver, reuseHandle } = fixture();
  const { ref, observed } = await readExisting(session);
  reuseHandle();
  const before = vi.mocked(driver.call).mock.calls.length;
  expect((await session.act({ window_ref: ref, observation_id: observed.details.observation_id!,
    action: "type_text", text: "Do not dispatch", element_token: "s1:0" })).details.status).toBe("refused");
  expect((await session.observe(ref)).details.status).toBe("refused");
  expect(driver.call).toHaveBeenCalledTimes(before);
});

test("dispatch consumes its observation even when the native outcome is unknown", async () => {
  const { session, driver } = fixture();
  const { ref, observed } = await readExisting(session);
  vi.mocked(driver.call).mockRejectedValueOnce(new Error("transport lost"));
  const args = { window_ref: ref, observation_id: observed.details.observation_id!,
    action: "type_text" as const, text: "Hello", element_token: "s1:0" };
  expect((await session.act(args)).details.status).toBe("unknown");
  expect(driver.close).toHaveBeenCalledOnce();
  const before = vi.mocked(driver.call).mock.calls.length;
  expect((await session.act(args)).details.status).toBe("refused");
  expect(driver.call).toHaveBeenCalledTimes(before);
});

test("a cancelled action does not dispatch or consume its observation", async () => {
  const { session } = fixture();
  const { ref, observed } = await readExisting(session);
  const args = { window_ref: ref, observation_id: observed.details.observation_id!,
    action: "type_text" as const, text: "Hello", element_token: "s1:0" };
  const controller = new AbortController(); controller.abort();
  expect((await session.act(args, controller.signal)).details.status).toBe("refused");
  expect((await session.act(args)).details.status).toBe("delivered");
});

test("a fresh read retires earlier tokens and close invalidates all window references", async () => {
  const { session } = fixture();
  const { ref, observed } = await readExisting(session);
  await session.observe(ref);
  expect((await session.act({ window_ref: ref, observation_id: observed.details.observation_id!,
    action: "set_value", text: "No", element_token: "s1:0" })).details.status).toBe("refused");
  await session.close();
  expect((await session.observe(ref)).details.status).toBe("refused");
});