Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

4.5 KB
Raw Blame
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import path from "node:path";
import { assistantMessage, getToolExecution, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";
import { expect, test } from "vitest";
import { IDE_EXTENSION, PI_COMMAND } from "./fixtures/pi-agent-ide.ts";

const root = path.resolve(import.meta.dirname, "..");
const fixture = path.join(root, "test/fixtures/window-management-extension.ts");
const driver = path.join(root, "test/fixtures/window-management-driver.mjs");

// Pattern: executable real-Pi extension example and the ordinary-IDE restore suite.
// Contract: computer_window samples native desktop geometry for one command; it is never pixel/input authority.
for (const scenario of ["workflow", "admission", "receipts", "restore", "typed"] as const) test.each([false, true])(
  `ordinary IDE window management protects ${scenario} (providerFirst=%s)`,
  async providerFirst => {
    const extension = IDE_EXTENSION;
    await mkdir(path.join(root, ".tmp/lpt-731"), { recursive: true });
    const cwd = await mkdtemp(path.join(root, ".tmp/lpt-731/pi-management-"));
    try {
      await mkdir(path.join(cwd, ".pi"));
      await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({ command: process.execPath, args: [driver, path.join(cwd, "calls.jsonl")] }));
      const result = await new PiIntegrationTest({
        testName: `ide-window-management-${scenario}-${providerFirst ? "provider-first" : "core-first"}`,
        artifactsDir: testArtifactsDir(import.meta.filename), cwd, isolateUserResources: true, rawMode: false,
        piCommand: PI_COMMAND,
        extensions: providerFirst ? [fixture, extension] : [extension, fixture],
        tools: ["read", "search", "write", "insert", "delete", "verify_window_management"], timeoutMs: 90_000,
        conversation: [assistantMessage([toolCall({ id: "management-proof", name: "verify_window_management", arguments: { scenario } })], { stopReason: "toolUse" }),
          assistantMessage([text("Finished the synthetic window management scenario.")])],
      }).run("Use ordinary IDE tools against the synthetic MCP window fixture only. Never operate a real desktop or use input fallback.");
      expect(result.tuiRenderedOutput, "Extension loading must succeed before evaluating behavior").not.toContain("Failed to load extension");
      expect(getToolExecution(result, "management-proof").isError, getToolResultText(result, "management-proof")).toBe(false);
      expect(getToolResultText(result, "management-proof")).toContain(`IDE window management ${scenario} contract passed`);
      const rows = (await readFile(path.join(cwd, "calls.jsonl"), "utf8")).trim().split("\n").filter(Boolean).map(line => JSON.parse(line));
      const mutations = rows.filter(row => row.name === "manage_window");
      expect(mutations).toHaveLength({ workflow: 5, admission: 1, receipts: 16, restore: 0, typed: 3 }[scenario]);
      for (const { arguments: args } of mutations) {
        const fields = args.operation === "move" ? ["x", "y", "unit"] : args.operation === "resize" ? ["width", "height", "unit"] : [];
        expect(Object.keys(args).sort()).toEqual(["pid", "window_id", "state_id", "operation", ...fields].sort());
        expect(args).toMatchObject({ pid: 42, window_id: expect.any(Number), state_id: expect.stringMatching(/^state-/) });
        if (fields.length) expect(args.unit).toBe("windows_desktop_pixels");
      }
      if (scenario === "workflow") {
        expect(mutations.map(row => row.arguments.operation)).toEqual(["move", "resize", "maximize", "minimize", "move"]);
        expect(mutations[0].arguments).toMatchObject({ x: -20, y: 120 });
        expect(mutations[1].arguments).toMatchObject({ width: 180, height: 120 });
      }
      const restores = rows.filter(row => row.name === "restore_window");
      if (scenario === "typed") expect(mutations.map(row => row.arguments.operation)).toEqual(["move", "resize", "maximize"]);
      expect(restores).toHaveLength(scenario === "restore" ? 2 : scenario === "typed" ? 1 : 0);
      for (const row of restores) expect(Object.keys(row.arguments).sort()).toEqual(["pid", "state_id", "window_id"]);
      expect(rows.filter(row => ["click", "type_text", "hotkey", "press_key", "bring_to_front", "set_window_frame", "move_pointer", "scroll_at", "get_desktop_state", "zoom"].includes(row.name))).toEqual([]);
      expect(result.tuiRenderedOutput).not.toContain("already handles");
    } finally { await rm(cwd, { recursive: true, force: true }); }
  }, 120_000,
);