Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

4.1 KB
Raw Blame
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import path from "node:path";
import {
  assistantMessage, getToolCallNames, getToolExecution, getToolExecutionDetails,
  getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall,
} from "pi-coding-agent-test";
import { expect, test } from "vitest";

type Action = { metadata: { semanticAction: Record<string, unknown> } };
const action = (details: unknown) => (details as Action).metadata.semanticAction;
const open = (id: string) => toolCall({ id, name: "computer_open", arguments: { at: "computer:machine/local", app: "Notepad" } });

test("standalone Pi opens two windows with first observations, rejects invented refs, and releases MCP", async () => {
  await mkdir(".tmp", { recursive: true });
  const cwd = await mkdtemp(path.resolve(".tmp", "standalone-contract-"));
  const log = path.join(cwd, "driver-calls.jsonl");
  await mkdir(path.join(cwd, ".pi"));
  await writeFile(path.join(cwd, ".pi", "computer-use.json"), JSON.stringify({
    command: process.execPath, args: [path.resolve("test/fixtures/fake-driver.mjs"), log],
  }));
  try {
    const result = await new PiIntegrationTest({
      testName: "standalone-new-window-and-refusal", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
      extensions: [path.resolve("test/fixtures/fake-extension.ts")],
      tools: ["computer_open", "computer_type", "computer_delete"], isolateUserResources: true,
      conversation: [
        assistantMessage([open("first")], { stopReason: "toolUse" }),
        assistantMessage([open("second")], { stopReason: "toolUse" }),
        assistantMessage([toolCall({ id: "stale", name: "computer_type", arguments: {
          window: "computer:window/not-a-window", text: "Do not dispatch",
        } })], { stopReason: "toolUse" }),
        assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
        assistantMessage([text("The standalone contract was exercised.")]),
      ],
    }).run("Open two new test windows with first observations, refuse an invented window reference, and release runtime.");
    expect(getToolCallNames(result)).toEqual(["computer_open", "computer_open", "computer_type", "computer_delete"]);
    const first = action(getToolExecutionDetails(getToolExecution(result, "first")));
    const second = action(getToolExecutionDetails(getToolExecution(result, "second")));
    expect(first).toMatchObject({ status: "observed", opened: true, mode: "yolo", observation_id: expect.any(String),
      target: { pid: 42, window_id: 101 }, state: { elements: [{ element_token: "s1:0" }] } });
    expect(second).toMatchObject({ status: "observed", target: { pid: 42, window_id: 102 } });
    expect(second.window_ref).not.toBe(first.window_ref);
    // The text shows this run's short addresses (LPT-932); the long references stay in the details.
    const sources = [...new Set(getToolResultText(result, "first").match(/computer:local\/[wo]\d+[a-z]{2}\b/g))];
    expect(sources).toHaveLength(2);
    expect(sources).toEqual(expect.arrayContaining([
      expect.stringMatching(/^computer:local\/w\d+[a-z]{2}$/), expect.stringMatching(/^computer:local\/o\d+[a-z]{2}$/),
    ]));
    expect(getToolExecution(result, "first").isError).toBe(false);
    expect(getToolExecution(result, "stale").isError).toBe(true);
    expect(action(getToolExecutionDetails(getToolExecution(result, "stale")))).toMatchObject({ ok: false, effect: "not-applied" });
    const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
    expect(calls.filter(call => call.name === "launch_app")).toHaveLength(2);
    expect(calls.filter(call => call.name === "get_window_state")).toHaveLength(2);
    expect(calls.filter(call => call.name === "type_text")).toHaveLength(0);
    expect(calls).toContainEqual({ event: "stdin-eof" });
    expect(action(getToolExecutionDetails(getToolExecution(result, "close")))).toMatchObject({ status: "closed", effect: "applied", ok: true });
    expect(getToolExecution(result, "close").isError).toBe(false);
  } finally { await rm(cwd, { recursive: true, force: true }); }
});