Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

14.4 KB
Raw Blame
import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
import path from "node:path";
import { expect, test } from "vitest";
import { McpDriver } from "../src/driver.ts";
import type { BundledRuntime } from "../src/native-runtime.ts";

test("the desktop profile inherits every earlier window capability and checks its bytes before starting", async () => {
  const backend = new McpDriver({ command: process.execPath, runtime: "windows-desktop" });
  try {
    expect([backend.windowManagement, backend.windowRestore, backend.coordinateScroll, backend.nativePointer,
      backend.heldPaths, backend.heldKeys, backend.selectedDisplays, backend.captureBoundDrag, backend.semanticActions]).toEqual(Array(9).fill(true));
    expect(backend.overviewKernel).toBe("bilinear");
    await expect(backend.call("list_windows", {})).rejects.toThrow("artifact hash mismatch");
  } finally { await backend.close(); }
});
test("management preserves inherited routes but rejects changed bytes before state acquisition", async () => {
  const backend = new McpDriver({ command: process.execPath, runtime: "windows-window-management" });
  try {
    expect([backend.windowManagement, backend.windowRestore, backend.coordinateScroll, backend.nativePointer,
      backend.heldPaths, backend.heldKeys, backend.selectedDisplays, backend.captureBoundDrag, backend.semanticActions]).toEqual(Array(9).fill(true));
    expect(backend.overviewKernel).toBe("bilinear");
    await expect(backend.call("get_window_control_state", { pid: 42, window_id: 91 })).rejects.toThrow("artifact hash mismatch");
  } finally { await backend.close(); }
});
test("old profiles cannot dispatch management or acquire its admission through direct calls", async () => {
  for (const runtime of [undefined, "windows-persistent-wgc", "windows-coordinate-scroll", "windows-window-restore"] as const) {
    const backend = new McpDriver({ command: process.execPath, runtime });
    try {
      expect(backend.windowManagement).toBe(false);
      for (const name of ["get_window_control_state", "manage_window"])
        expect(await backend.call(name, { pid: 42, window_id: 91, state_id: "state", operation: "focus" })).toMatchObject({
          isError: true, structuredContent: { code: "window_management_unavailable", mutation_attempted: false, mutation_accepted: false, postcondition_confirmed: false },
        });
    } finally { await backend.close(); }
  }
});

test("the shipped Windows runtime gets the desktop capabilities and refuses a missing package before any native call", async () => {
  const bundled: BundledRuntime = { target: "win32-x64", root: "/missing-package", command: "/missing-package/runtime/cua-driver.exe",
    source: "a".repeat(40), version: "0.37.0", driverVersion: "0.37.0", files: {} };
  const backend = new McpDriver({ command: bundled.command, bundled, runtime: "windows-desktop" });
  try {
    expect(backend.platform).toBe("windows");
    expect(backend.windowLifetime).toBe(true);
    expect([backend.windowManagement, backend.windowRestore, backend.coordinateScroll, backend.nativePointer, backend.heldPaths,
      backend.heldKeys, backend.selectedDisplays, backend.captureBoundDrag, backend.semanticActions]).toEqual(Array(9).fill(true));
    await expect(backend.call("list_windows", {})).rejects.toThrow("native_runtime_files_mismatch");
  } finally { await backend.close(); }
});

test.each(["windows-held-paths", "windows-key-holds"] as const)("an unqualified %s executable cannot start a peer or dispatch input", async runtime => {
  const backend = new McpDriver({ command: process.execPath, runtime });
  try { await expect(backend.call("list_windows", {})).rejects.toThrow("artifact hash mismatch"); }
  finally { await backend.close(); }
});
test("older runtimes refuse key holds without starting an unqualified peer", async () => {
  for (const runtime of [undefined, "windows-held-paths"] as const) {
    const backend = new McpDriver({ command: process.execPath, runtime });
    try {
      expect(backend.heldKeys).toBe(false);
      expect(await backend.call("hold_keys", { keys: ["w"], duration_ms: 500 })).toMatchObject({
        isError: true, structuredContent: { effect: "refused", code: "key_hold_unavailable" },
      });
    } finally { await backend.close(); }
  }
});
test("older profiles refuse capture-bound drag before starting any native peer", async () => {
  for (const runtime of [undefined, "windows-held-paths", "windows-key-holds", "windows-selected-displays"] as const) {
    const backend = new McpDriver({ command: process.execPath, runtime });
    try {
      expect(backend.captureBoundDrag).toBe(false);
      expect(await backend.call("drag", { capture_id: "unqualified" })).toMatchObject({
        isError: true, structuredContent: { effect: "refused", code: "capture_bound_drag_unavailable" },
      });
    } finally { await backend.close(); }
  }
});
test("an explicit test override can run a different driver version", async () => {
  await mkdir(".tmp/native-zoom", { recursive: true });
  const cwd = await mkdtemp(path.resolve(".tmp/native-zoom/override-"));
  const log = path.join(cwd, "calls.jsonl");
  const backend = new McpDriver({ command: process.execPath, args: [path.resolve("test/fixtures/detail-driver.mjs"), log, "0.39.0", "advertise"], driverOverride: true });
  try {
    const result = await backend.call("get_desktop_state", { max_image_dimension: 500 });
    expect(result.structuredContent).toBeDefined();
    expect(await readFile(log, "utf8")).not.toBe("");
  } finally { await backend.close(); await rm(cwd, { recursive: true, force: true }); }
});
test("the qualified pin rejects an older peer before native dispatch", async () => {
  await mkdir(".tmp/native-zoom", {recursive:true});
  const cwd = await mkdtemp(path.resolve(".tmp/native-zoom/version-")); const log = path.join(cwd,"calls.jsonl");
  const backend = new McpDriver({command:process.execPath,args:[path.resolve("test/fixtures/detail-driver.mjs"),log,"0.33.1"]});
  try {
    await expect(backend.call("get_desktop_state",{max_image_dimension:500})).rejects.toThrow("requires cua-driver 0.33.2");
    expect(await readFile(log,"utf8").catch(()=>"")).toBe("");
  } finally { await backend.close(); await rm(cwd,{recursive:true,force:true}); }
});

test("older native runtimes refuse selected-display reads before starting any peer", async () => {
  for (const runtime of [undefined, "windows-held-paths", "windows-key-holds"] as const) {
    const backend = new McpDriver({ command: process.execPath, runtime });
    try {
      expect(backend.selectedDisplays).toBe(false);
      for (const name of ["list_displays", "get_display_state"]) expect(await backend.call(name, {})).toMatchObject({
        isError: true, structuredContent: { effect: "refused", code: "selected_display_unavailable" },
      });
    } finally { await backend.close(); }
  }
});

test.each(["windows-shell-desktops", "windows-single-png"] as const)("the %s profile preserves earlier qualified capabilities and rejects wrong bytes before any membership read", async runtime => {
  const backend = new McpDriver({ command: process.execPath, runtime });
  try {
    expect([backend.heldPaths, backend.heldKeys, backend.selectedDisplays, backend.captureBoundDrag]).toEqual([true, true, true, true]);
    await expect(backend.call("list_windows", {})).rejects.toThrow("artifact hash mismatch");
  } finally { await backend.close(); }
});
test("Bilinear qualification preserves semantic/input capabilities and refuses wrong bytes before startup", async () => {
  const backend = new McpDriver({ command: process.execPath, runtime: "windows-fast-bilinear" });
  try {
    expect(backend.overviewKernel).toBe("bilinear");
    expect([backend.heldPaths, backend.heldKeys, backend.selectedDisplays, backend.captureBoundDrag, backend.semanticActions]).toEqual([true, true, true, true, true]);
    await expect(backend.call("list_windows", {})).rejects.toThrow("artifact hash mismatch");
  } finally { await backend.close(); }
});
test("direct lookup keeps Bilinear and native capabilities but refuses changed artifact bytes", async () => {
  const backend = new McpDriver({ command: process.execPath, runtime: "windows-direct-process" });
  try {
    expect(backend.overviewKernel).toBe("bilinear");
    expect([backend.heldPaths, backend.heldKeys, backend.selectedDisplays, backend.captureBoundDrag, backend.semanticActions]).toEqual([true, true, true, true, true]);
    await expect(backend.call("list_windows", {})).rejects.toThrow("artifact hash mismatch");
  } finally { await backend.close(); }
});
test("existing profiles keep Lanczos3 overview metadata", () => {
  for (const runtime of [undefined, "windows-semantic-actions", "windows-single-png"] as const)
    expect(new McpDriver({ command: process.execPath, runtime }).overviewKernel).toBe("lanczos3");
});
test("strict semantic qualification preserves earlier capabilities and rejects wrong bytes before UIA dispatch", async () => {
  const backend = new McpDriver({ command: process.execPath, runtime: "windows-semantic-actions" });
  try {
    expect([backend.heldPaths, backend.heldKeys, backend.selectedDisplays, backend.captureBoundDrag, backend.semanticActions]).toEqual([true, true, true, true, true]);
    await expect(backend.call("semantic_action", { operation: "invoke" })).rejects.toThrow("artifact hash mismatch");
  } finally { await backend.close(); }
});
test("the capture-bound drag profile preserves earlier capabilities but rejects wrong bytes before startup", async () => {
  const backend = new McpDriver({ command: process.execPath, runtime: "windows-capture-drag" });
  try {
    expect([backend.heldPaths, backend.heldKeys, backend.selectedDisplays, backend.captureBoundDrag]).toEqual([true, true, true, true]);
    await expect(backend.call("drag", { capture_id: "candidate" })).rejects.toThrow("artifact hash mismatch");
  } finally { await backend.close(); }
});
test("the exact selected-display profile preserves keys and paths but rejects wrong bytes before startup", async () => {
  const backend = new McpDriver({ command: process.execPath, runtime: "windows-selected-displays" });
  try {
    expect([backend.heldPaths, backend.heldKeys, backend.selectedDisplays]).toEqual([true, true, true]);
    await expect(backend.call("list_displays", {})).rejects.toThrow("artifact hash mismatch");
  } finally { await backend.close(); }
});

test("the persistent-WGC runtime profile preserves earlier capabilities but refuses wrong bytes before capture", async () => {
  const backend = new McpDriver({ command: process.execPath, runtime: "windows-persistent-wgc" });
  try {
    expect(backend.overviewKernel).toBe("bilinear");
    expect([backend.heldPaths, backend.heldKeys, backend.selectedDisplays, backend.captureBoundDrag, backend.semanticActions]).toEqual([true, true, true, true, true]);
    await expect(backend.call("get_window_state", {})).rejects.toThrow("artifact hash mismatch");
  } finally { await backend.close(); }
});
test("native pointer candidate keeps the capture base but rejects changed bytes before movement", async () => {
  const backend = new McpDriver({ command: process.execPath, runtime: "windows-native-pointer" });
  try {
    expect(backend.overviewKernel).toBe("bilinear");
    expect([backend.heldPaths, backend.heldKeys, backend.selectedDisplays, backend.captureBoundDrag,
      backend.semanticActions, backend.nativePointer]).toEqual([true, true, true, true, true, true]);
    await expect(backend.call("move_pointer", { delivery_mode: "foreground" })).rejects.toThrow("artifact hash mismatch");
  } finally { await backend.close(); }
});
test("restore candidate inherits qualified capabilities and rejects wrong bytes before state or mutation", async () => {
  const backend = new McpDriver({ command: process.execPath, runtime: "windows-window-restore" });
  try {
    expect(backend.overviewKernel).toBe("bilinear");
    expect([backend.heldPaths, backend.heldKeys, backend.selectedDisplays, backend.captureBoundDrag,
      backend.semanticActions, backend.nativePointer, backend.coordinateScroll, backend.windowRestore])
      .toEqual([true, true, true, true, true, true, true, true]);
    for (const name of ["get_window_status", "restore_window"])
      await expect(backend.call(name, { pid: 42, window_id: 91, state_id: "state" })).rejects.toThrow("artifact hash mismatch");
  } finally { await backend.close(); }
});
test("older runtime profiles refuse restore metadata and mutation without starting a peer", async () => {
  for (const runtime of [undefined, "windows-persistent-wgc", "windows-native-pointer", "windows-coordinate-scroll"] as const) {
    const backend = new McpDriver({ command: process.execPath, runtime });
    try {
      expect(backend.windowRestore).toBe(false);
      for (const name of ["get_window_status", "restore_window"])
        expect(await backend.call(name, { pid: 42, window_id: 91, state_id: "state" })).toMatchObject({
          isError: true, structuredContent: { code: "window_restore_unavailable", effect: "refused", fallback: false,
            restore_attempted: false, restore_requested: false, restored: false, activation_attempted: false, activation_succeeded: false },
        });
    } finally { await backend.close(); }
  }
});
test("coordinate scroll keeps inherited Windows capabilities but rejects changed bytes before input", async () => {
  const backend = new McpDriver({ command: process.execPath, runtime: "windows-coordinate-scroll" });
  try {
    expect(backend.overviewKernel).toBe("bilinear");
    expect([backend.heldPaths, backend.heldKeys, backend.selectedDisplays, backend.captureBoundDrag,
      backend.semanticActions, backend.nativePointer, backend.coordinateScroll])
      .toEqual([true, true, true, true, true, true, true]);
    await expect(backend.call("scroll_at", { delivery_mode: "foreground" })).rejects.toThrow("artifact hash mismatch");
  } finally { await backend.close(); }
});
test("older runtimes refuse coordinate scroll before starting any native peer", async () => {
  for (const runtime of [undefined, "windows-persistent-wgc", "windows-native-pointer"] as const) {
    const backend = new McpDriver({ command: process.execPath, runtime });
    try {
      expect(backend.coordinateScroll).toBe(false);
      expect(await backend.call("scroll_at", { delivery_mode: "foreground" })).toMatchObject({
        isError: true, structuredContent: { code: "coordinate_scroll_unavailable", attempted: false, delivered: false,
          pointer_attempted: false, pointer_delivered: false, attempted_axes: [], delivered_axes: [], fallback: false },
      });
    } finally { await backend.close(); }
  }
});