Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Finish writing tests for interfaces
Morgan Dean613745da7f11
diff --git a/libs/computer/typescript/package.json b/libs/computer/typescript/package.jsonindex 8ff5c0953e..71141f22cc 100644--- a/libs/computer/typescript/package.json+++ b/libs/computer/typescript/package.json@@ -48,7 +48,6 @@"@types/ws": "^8.18.1","bumpp": "^10.1.0","happy-dom": "^17.4.7",- "msw": "^2.10.2","tsdown": "^0.11.9","tsx": "^4.19.4","typescript": "^5.8.3",diff --git a/libs/computer/typescript/pnpm-lock.yaml b/libs/computer/typescript/pnpm-lock.yamlindex 1abeebdcdd..9104a636bb 100644--- a/libs/computer/typescript/pnpm-lock.yaml+++ b/libs/computer/typescript/pnpm-lock.yaml@@ -33,9 +33,6 @@ importers:happy-dom:specifier: ^17.4.7version: 17.6.3- msw:- specifier: ^2.10.2- version: 2.10.2(@types/[email protected])([email protected])tsdown:specifier: ^0.11.9version: 0.11.13([email protected])@@ -1420,15 +1417,18 @@ snapshots:'@bundled-es-modules/[email protected]':dependencies:cookie: 0.7.2+ optional: true'@bundled-es-modules/[email protected]':dependencies:statuses: 2.0.2+ optional: true'@bundled-es-modules/[email protected]':dependencies:'@types/tough-cookie': 4.0.5tough-cookie: 4.1.4+ optional: true'@emnapi/[email protected]':dependencies:@@ -1602,6 +1602,7 @@ snapshots:'@inquirer/type': 3.0.7(@types/[email protected])optionalDependencies:'@types/node': 22.15.31+ optional: true'@inquirer/[email protected](@types/[email protected])':dependencies:@@ -1615,12 +1616,15 @@ snapshots:yoctocolors-cjs: 2.1.2optionalDependencies:'@types/node': 22.15.31+ optional: true- '@inquirer/[email protected]': {}+ '@inquirer/[email protected]':+ optional: true'@inquirer/[email protected](@types/[email protected])':optionalDependencies:'@types/node': 22.15.31+ optional: true'@jridgewell/[email protected]':dependencies:@@ -1647,6 +1651,7 @@ snapshots:is-node-process: 1.2.0outvariant: 1.4.3strict-event-emitter: 0.5.1+ optional: true'@napi-rs/[email protected]':dependencies:@@ -1655,14 +1660,17 @@ snapshots:'@tybys/wasm-util': 0.9.0optional: true- '@open-draft/[email protected]': {}+ '@open-draft/[email protected]':+ optional: true'@open-draft/[email protected]':dependencies:is-node-process: 1.2.0outvariant: 1.4.3+ optional: true- '@open-draft/[email protected]': {}+ '@open-draft/[email protected]':+ optional: true'@oxc-project/[email protected]': {}@@ -1779,7 +1787,8 @@ snapshots:dependencies:'@types/deep-eql': 4.0.2- '@types/[email protected]': {}+ '@types/[email protected]':+ optional: true'@types/[email protected]':dependencies:@@ -1799,9 +1808,11 @@ snapshots:dependencies:undici-types: 6.21.0- '@types/[email protected]': {}+ '@types/[email protected]':+ optional: true- '@types/[email protected]': {}+ '@types/[email protected]':+ optional: true'@types/[email protected]':dependencies:@@ -1853,12 +1864,15 @@ snapshots:dependencies:type-fest: 0.21.3+ optional: true- [email protected]: {}+ optional: truedependencies:color-convert: 2.0.1+ optional: true@@ -1926,13 +1940,15 @@ snapshots:dependencies:consola: 3.4.2- [email protected]: {}+ optional: truedependencies:string-width: 4.2.3strip-ansi: 6.0.1wrap-ansi: 7.0.0+ optional: truedependencies:@@ -1954,7 +1970,8 @@ snapshots:- [email protected]: {}+ optional: truedependencies:@@ -1974,7 +1991,8 @@ snapshots:- [email protected]: {}+ optional: true@@ -2027,7 +2045,8 @@ snapshots:optional: true- [email protected]: {}+ optional: truedependencies:@@ -2042,22 +2061,26 @@ snapshots:nypm: 0.6.0pathe: 2.0.3- [email protected]: {}+ optional: truedependencies:webidl-conversions: 7.0.0whatwg-mimetype: 3.0.0- [email protected]: {}+ optional: true- [email protected]: {}+ optional: true- [email protected]: {}+ optional: true@@ -2099,8 +2122,10 @@ snapshots:typescript: 5.8.3transitivePeerDependencies:- '@types/node'+ optional: true- [email protected]: {}+ optional: true@@ -2118,11 +2143,13 @@ snapshots:- [email protected]: {}+ optional: true- [email protected]: {}+ optional: true@@ -2171,12 +2198,15 @@ snapshots:dependencies:punycode: 2.3.1+ optional: true- [email protected]: {}+ optional: true- [email protected]: {}+ optional: true@@ -2189,9 +2219,11 @@ snapshots:- [email protected]: {}+ optional: true- [email protected]: {}+ optional: true@@ -2289,7 +2321,8 @@ snapshots:- [email protected]: {}+ optional: truedependencies:@@ -2305,21 +2338,25 @@ snapshots:- [email protected]: {}+ optional: true- [email protected]: {}+ optional: truedependencies:emoji-regex: 8.0.0is-fullwidth-code-point: 3.0.0strip-ansi: 6.0.1+ optional: truedependencies:ansi-regex: 5.0.1+ optional: truedependencies:@@ -2352,6 +2389,7 @@ snapshots:punycode: 2.3.1universalify: 0.2.0url-parse: 1.5.10+ optional: truedependencies:@@ -2387,9 +2425,11 @@ snapshots:optionalDependencies:fsevents: 2.3.3- [email protected]: {}+ optional: true- [email protected]: {}+ optional: true@@ -2402,12 +2442,14 @@ snapshots:- [email protected]: {}+ optional: truedependencies:querystringify: 2.2.0requires-port: 1.0.0+ optional: true[email protected](@types/[email protected])([email protected])([email protected])([email protected]):dependencies:@@ -2502,20 +2544,24 @@ snapshots:ansi-styles: 4.3.0string-width: 4.2.3strip-ansi: 6.0.1+ optional: truedependencies:ansi-styles: 4.3.0string-width: 4.2.3strip-ansi: 6.0.1+ optional: true- [email protected]: {}+ optional: true- [email protected]: {}+ optional: truedependencies:@@ -2526,5 +2572,7 @@ snapshots:string-width: 4.2.3y18n: 5.0.8yargs-parser: 21.1.1+ optional: true- [email protected]: {}+ optional: truediff --git a/libs/computer/typescript/src/interface/base.ts b/libs/computer/typescript/src/interface/base.tsindex 169cf4b0c9..6455fb29ed 100644--- a/libs/computer/typescript/src/interface/base.ts+++ b/libs/computer/typescript/src/interface/base.ts@@ -39,6 +39,7 @@ export abstract class BaseComputerInterface {protected ws: WebSocket;protected apiKey?: string;protected vmName?: string;+ protected secure?: boolean;protected logger = pino({ name: "interface-base" });@@ -47,13 +48,15 @@ export abstract class BaseComputerInterface {username = "lume",password = "lume",apiKey?: string,- vmName?: string+ vmName?: string,+ secure?: boolean) {this.ipAddress = ipAddress;this.username = username;this.password = password;this.apiKey = apiKey;this.vmName = vmName;+ this.secure = secure;// Initialize WebSocket with headers if neededconst headers: { [key: string]: string } = {};@@ -71,9 +74,15 @@ export abstract class BaseComputerInterface {* Subclasses can override this to customize the URI.*/protected get wsUri(): string {- // Use secure WebSocket for cloud provider with API key- const protocol = this.apiKey ? "wss" : "ws";- const port = this.apiKey ? "8443" : "8000";+ const protocol = this.secure ? "wss" : "ws";++ // Check if ipAddress already includes a port+ if (this.ipAddress.includes(":")) {+ return `${protocol}://${this.ipAddress}/ws`;+ }++ // Otherwise, append the default port+ const port = this.secure ? "8443" : "8000";return `${protocol}://${this.ipAddress}:${port}/ws`;}diff --git a/libs/computer/typescript/tests/interface/integration.test.ts b/libs/computer/typescript/tests/interface/integration.test.tsdeleted file mode 100644index b0867ca7c4..0000000000--- a/libs/computer/typescript/tests/interface/integration.test.ts+++ /dev/null@@ -1,447 +0,0 @@-import {- describe,- expect,- it,- beforeEach,- afterEach,- vi,- beforeAll,- afterAll,-} from "vitest";-import { InterfaceFactory } from "../../src/interface/factory.ts";-import { OSType } from "../../src/types.ts";-import { ws } from "msw";-import { setupServer } from "msw/node";--describe("Interface Integration Tests", () => {- const testIp = "192.168.1.100";- const testPort = 8000;-- // Create WebSocket server- const server = setupServer();-- beforeAll(() => {- server.listen({ onUnhandledRequest: "error" });- });-- afterAll(() => {- server.close();- });-- beforeEach(() => {- // Reset handlers for each test- server.resetHandlers();- });-- afterEach(() => {- vi.clearAllMocks();- });-- describe("Cross-platform interface creation", () => {- it("should create correct interface for each OS type", async () => {- const osTypes = [OSType.MACOS, OSType.LINUX, OSType.WINDOWS];- const interfaces: Array<{- os: OSType;- interface: ReturnType<typeof InterfaceFactory.createInterfaceForOS>;- }> = [];-- // Create interfaces for each OS- for (const os of osTypes) {- const interface_ = InterfaceFactory.createInterfaceForOS(os, testIp);- interfaces.push({ os, interface: interface_ });- }-- // Verify each interface is created correctly- expect(interfaces).toHaveLength(3);- for (const { os, interface: iface } of interfaces) {- expect(iface).toBeDefined();- // Check that the interface name contains the OS type in some form- const osName = os.toLowerCase();- expect(iface.constructor.name.toLowerCase()).toContain(osName);- }- });-- it("should handle multiple interfaces with different IPs", async () => {- const ips = ["192.168.1.100", "192.168.1.101", "192.168.1.102"];- const interfaces = ips.map((ip) =>- InterfaceFactory.createInterfaceForOS(OSType.MACOS, ip)- );-- // Set up WebSocket handlers for each IP- for (const ip of ips) {- const wsLink = ws.link(`ws://${ip}:${testPort}/ws`);- server.use(- wsLink.addEventListener("connection", ({ client }) => {- client.addEventListener("message", () => {- // Echo back success response- client.send(JSON.stringify({ success: true }));- });- })- );- }-- // Connect all interfaces- await Promise.all(interfaces.map((iface) => iface.connect()));-- // Verify all are connected- for (const iface of interfaces) {- expect(iface.isConnected()).toBe(true);- }-- // Clean up- for (const iface of interfaces) {- iface.disconnect();- }- });- });-- describe("Connection management", () => {- it("should handle connection lifecycle", async () => {- // Set up WebSocket handler- const wsLink = ws.link(`ws://${testIp}:${testPort}/ws`);- server.use(- wsLink.addEventListener("connection", ({ client }) => {- client.addEventListener("message", () => {- // Echo back success response- client.send(JSON.stringify({ success: true }));- });- })- );-- const interface_ = InterfaceFactory.createInterfaceForOS(- OSType.MACOS,- testIp- );-- // Initially not connected- expect(interface_.isConnected()).toBe(false);-- // Connect- await interface_.connect();- expect(interface_.isConnected()).toBe(true);-- // Disconnect- interface_.disconnect();-- // Wait a tick for the close to process- await new Promise((resolve) => process.nextTick(resolve));- expect(interface_.isConnected()).toBe(false);- });-- it("should handle connection errors gracefully", async () => {- // Don't register a handler - connection will succeed but no responses- const interface_ = InterfaceFactory.createInterfaceForOS(- OSType.MACOS,- "192.0.2.1" // TEST-NET-1 address- );-- // Should connect (WebSocket mock always connects)- await interface_.connect();- expect(interface_.isConnected()).toBe(true);-- interface_.disconnect();- });-- it("should handle secure connections", async () => {- const secureIp = "192.0.2.1";- const securePort = 8443;-- // Register handler for secure connection- const wsLink = ws.link(`wss://${secureIp}:${securePort}/ws`);- server.use(- wsLink.addEventListener("connection", ({ client }) => {- client.addEventListener("message", () => {- // Echo back success response- client.send(JSON.stringify({ success: true }));- });- })- );-- const interface_ = InterfaceFactory.createInterfaceForOS(- OSType.MACOS,- secureIp,- "testuser",- "testpass"- );-- await interface_.connect();- expect(interface_.isConnected()).toBe(true);-- interface_.disconnect();- });- });-- describe("Performance and concurrency", () => {- it("should handle rapid command sequences", async () => {- const receivedCommands: string[] = [];-- // Set up WebSocket handler that tracks commands- const wsLink = ws.link(`ws://${testIp}:${testPort}/ws`);- server.use(- wsLink.addEventListener("connection", ({ client }) => {- client.addEventListener("message", (event) => {- const data = JSON.parse(event.data as string);- receivedCommands.push(data.action);- // Send response with command index- client.send(- JSON.stringify({- success: true,- data: `Response for ${data.action}`,- })- );- });- })- );-- const interface_ = InterfaceFactory.createInterfaceForOS(- OSType.MACOS,- testIp- );- await interface_.connect();-- // Send multiple commands rapidly- const commands = ["left_click", "right_click", "double_click"];- const promises = commands.map((cmd) => {- switch (cmd) {- case "left_click":- return interface_.leftClick(100, 200);- case "right_click":- return interface_.rightClick(150, 250);- case "double_click":- return interface_.doubleClick(200, 300);- }- });-- await Promise.all(promises);-- // Verify all commands were received- expect(receivedCommands).toHaveLength(3);- expect(receivedCommands).toContain("left_click");- expect(receivedCommands).toContain("right_click");- expect(receivedCommands).toContain("double_click");-- interface_.disconnect();- });-- it("should maintain command order with locking", async () => {- const receivedCommands: Array<{ action: string; index: number }> = [];-- // Set up WebSocket handler that tracks commands with delay- const wsLink = ws.link(`ws://${testIp}:${testPort}/ws`);- server.use(- wsLink.addEventListener("connection", ({ client }) => {- client.addEventListener("message", async (event) => {- // Add delay to simulate processing- await new Promise((resolve) => setTimeout(resolve, 10));-- const data = JSON.parse(event.data as string);- receivedCommands.push({- action: data.action,- index: data.index,- });-- client.send(JSON.stringify({ success: true }));- });- })- );-- const interface_ = InterfaceFactory.createInterfaceForOS(- OSType.MACOS,- testIp- );- await interface_.connect();-- // Helper to send command with index- async function sendCommandWithIndex(action: string, index: number) {- await interface_.sendCommand({ action, index });- }-- // Send commands in sequence- await sendCommandWithIndex("command1", 0);- await sendCommandWithIndex("command2", 1);- await sendCommandWithIndex("command3", 2);-- // Wait for all commands to be processed- await new Promise((resolve) => setTimeout(resolve, 50));-- // Verify commands were received in order- expect(receivedCommands).toHaveLength(3);- expect(receivedCommands[0]).toEqual({ action: "command1", index: 0 });- expect(receivedCommands[1]).toEqual({ action: "command2", index: 1 });- expect(receivedCommands[2]).toEqual({ action: "command3", index: 2 });-- interface_.disconnect();- });- });-- describe("Error handling", () => {- it("should handle command failures", async () => {- // Set up WebSocket handler that returns errors- const wsLink = ws.link(`ws://${testIp}:${testPort}/ws`);- server.use(- wsLink.addEventListener("connection", ({ client }) => {- client.addEventListener("message", (event) => {- const data = JSON.parse(event.data as string);-- if (data.action === "fail_command") {- client.send(- JSON.stringify({- success: false,- error: "Command failed",- })- );- } else {- client.send(JSON.stringify({ success: true }));- }- });- })- );-- const interface_ = InterfaceFactory.createInterfaceForOS(- OSType.MACOS,- testIp- );- await interface_.connect();-- // Send a failing command- await expect(- interface_.sendCommand({ action: "fail_command" })- ).rejects.toThrow("Command failed");-- // Verify interface is still connected- expect(interface_.isConnected()).toBe(true);-- // Send a successful command- const result = await interface_.sendCommand({- action: "success_command",- });- expect(result.success).toBe(true);-- interface_.disconnect();- });-- it("should handle disconnection during command", async () => {- // Set up WebSocket handler that captures WebSocket instance- const wsLink = ws.link(`ws://${testIp}:${testPort}/ws`);- server.use(- wsLink.addEventListener("connection", ({ client }) => {- client.addEventListener("message", async () => {- // Simulate disconnection during command processing- await new Promise((resolve) => setTimeout(resolve, 10));- client.close();- });- })- );-- const interface_ = InterfaceFactory.createInterfaceForOS(- OSType.MACOS,- testIp- );- await interface_.connect();-- // Send command that will trigger disconnection- await expect(- interface_.sendCommand({ action: "disconnect_me" })- ).rejects.toThrow();-- // Wait for close to process- await new Promise((resolve) => setTimeout(resolve, 20));-- // Verify interface is disconnected- expect(interface_.isConnected()).toBe(false);- });- });-- describe("Feature-specific tests", () => {- it("should handle screenshot commands", async () => {- // Set up WebSocket handler- const wsLink = ws.link(`ws://${testIp}:${testPort}/ws`);- server.use(- wsLink.addEventListener("connection", ({ client }) => {- client.addEventListener("message", () => {- // Echo back success response with screenshot data- client.send(JSON.stringify({- success: true,- data: "base64encodedimage"- }));- });- })- );-- const interface_ = InterfaceFactory.createInterfaceForOS(- OSType.MACOS,- testIp- );- await interface_.connect();-- const screenshot = await interface_.screenshot();- expect(screenshot).toBeInstanceOf(Buffer);- expect(screenshot.toString("base64")).toBe("base64encodedimage");-- interface_.disconnect();- });-- it("should handle screen size queries", async () => {- // Set up WebSocket handler- const wsLink = ws.link(`ws://${testIp}:${testPort}/ws`);- server.use(- wsLink.addEventListener("connection", ({ client }) => {- client.addEventListener("message", () => {- // Echo back success response with screen size- client.send(JSON.stringify({- success: true,- data: { width: 1920, height: 1080 }- }));- });- })- );-- const interface_ = InterfaceFactory.createInterfaceForOS(- OSType.LINUX,- testIp- );- await interface_.connect();-- const screenSize = await interface_.getScreenSize();- expect(screenSize).toEqual({ width: 1920, height: 1080 });-- interface_.disconnect();- });-- it("should handle file operations", async () => {- // Set up WebSocket handler- const wsLink = ws.link(`ws://${testIp}:${testPort}/ws`);- server.use(- wsLink.addEventListener("connection", ({ client }) => {- client.addEventListener("message", () => {- // Echo back success response with file data- client.send(JSON.stringify({- success: true,- data: "file content"- }));- });- })- );-- const interface_ = InterfaceFactory.createInterfaceForOS(- OSType.WINDOWS,- testIp- );- await interface_.connect();-- // Test file exists- const exists = await interface_.fileExists("/test/file.txt");- expect(exists).toBe(true);-- // Test read text- const content = await interface_.readText("/test/file.txt");- expect(content).toBe("file content");-- // Test list directory- const files = await interface_.listDir("/test");- expect(files).toEqual(["file1.txt", "file2.txt", "dir1"]);-- interface_.disconnect();- });- });-});diff --git a/libs/computer/typescript/tests/interface/linux.test.ts b/libs/computer/typescript/tests/interface/linux.test.tsindex d5d8532bdd..c82627c70d 100644--- a/libs/computer/typescript/tests/interface/linux.test.ts+++ b/libs/computer/typescript/tests/interface/linux.test.ts@@ -4,7 +4,7 @@ import { MacOSComputerInterface } from "../../src/interface/macos.ts";describe("LinuxComputerInterface", () => {const testParams = {- ipAddress: "192.0.2.1", // TEST-NET-1 address (RFC 5737) - guaranteed not to be routable+ ipAddress: "test.cua.com", // TEST-NET-1 address (RFC 5737) - guaranteed not to be routableusername: "testuser",password: "testpass",apiKey: "test-api-key",diff --git a/libs/computer/typescript/tests/interface/macos.test.ts b/libs/computer/typescript/tests/interface/macos.test.tsindex fb7e4b0884..78d2d9d3a5 100644--- a/libs/computer/typescript/tests/interface/macos.test.ts+++ b/libs/computer/typescript/tests/interface/macos.test.ts@@ -1,36 +1,139 @@-import {- describe,- expect,- it,- beforeEach,- afterEach,-} from "vitest";+import { describe, expect, it, beforeEach, afterEach } from "vitest";import { MacOSComputerInterface } from "../../src/interface/macos.ts";-// Import the setup.ts which already has MSW configured-import "../setup.ts";+import { WebSocketServer, WebSocket } from "ws";describe("MacOSComputerInterface", () => {// Define test parametersconst testParams = {- ipAddress: "192.0.2.1", // TEST-NET-1 address (RFC 5737) - guaranteed not to be routable+ ipAddress: "localhost",username: "testuser",password: "testpass",apiKey: "test-api-key",vmName: "test-vm",};- // Track received messages for verification- // biome-ignore lint/suspicious/noExplicitAny: <explanation>- let receivedMessages: any[] = [];+ // WebSocket server mock+ let wss: WebSocketServer;+ let serverPort: number;+ let connectedClients: WebSocket[] = [];- beforeEach(() => {- // Clear received messages before each test+ // Track received messages for verification+ interface ReceivedMessage {+ action: string;+ [key: string]: unknown;+ }+ let receivedMessages: ReceivedMessage[] = [];++ // Set up WebSocket server before all tests+ beforeEach(async () => {receivedMessages = [];+ connectedClients = [];++ // Create WebSocket server on a random available port+ wss = new WebSocketServer({ port: 0 });+ serverPort = (wss.address() as { port: number }).port;++ // Update test params with the actual server address+ testParams.ipAddress = `localhost:${serverPort}`;++ // Handle WebSocket connections+ wss.on("connection", (ws, req) => {+ connectedClients.push(ws);++ // Verify authentication headers+ const apiKey = req.headers["x-api-key"];+ const vmName = req.headers["x-vm-name"];++ if (apiKey !== testParams.apiKey || vmName !== testParams.vmName) {+ ws.close(1008, "Unauthorized");+ return;+ }++ // Handle incoming messages+ ws.on("message", (data) => {+ try {+ const message = JSON.parse(data.toString());+ receivedMessages.push(message);++ // Send appropriate responses based on action+ switch (message.action) {+ case "screenshot":+ ws.send(JSON.stringify({+ data: Buffer.from("fake-screenshot-data").toString("base64")+ }));+ break;+ case "get_screen_size":+ ws.send(JSON.stringify({ data: { width: 1920, height: 1080 } }));+ break;+ case "get_cursor_position":+ ws.send(JSON.stringify({ data: { x: 100, y: 200 } }));+ break;+ case "copy_to_clipboard":+ ws.send(JSON.stringify({ data: "clipboard content" }));+ break;+ case "file_exists":+ ws.send(JSON.stringify({ data: true }));+ break;+ case "directory_exists":+ ws.send(JSON.stringify({ data: true }));+ break;+ case "list_dir":+ ws.send(JSON.stringify({ data: ["file1.txt", "file2.txt"] }));+ break;+ case "read_text":+ ws.send(JSON.stringify({ data: "file content" }));+ break;+ case "read_bytes":+ ws.send(JSON.stringify({+ data: Buffer.from("binary content").toString("base64")+ }));+ break;+ case "run_command":+ ws.send(JSON.stringify({ data: { stdout: "command output", stderr: "" } }));+ break;+ case "get_accessibility_tree":+ ws.send(JSON.stringify({+ data: {+ role: "window",+ title: "Test Window",+ bounds: { x: 0, y: 0, width: 1920, height: 1080 },+ children: []+ }+ }));+ break;+ case "to_screen_coordinates":+ case "to_screenshot_coordinates":+ ws.send(JSON.stringify({ data: [message.x || 0, message.y || 0] }));+ break;+ default:+ // For all other actions, just send success+ ws.send(JSON.stringify({ success: true }));+ break;+ }+ } catch (error) {+ ws.send(JSON.stringify({ error: (error as Error).message }));+ }+ });++ ws.on("error", (error) => {+ console.error("WebSocket error:", error);+ });+ });});- afterEach(() => {- // Clear any state after each test- receivedMessages = [];+ // Clean up WebSocket server after each test+ afterEach(async () => {+ // Close all connected clients+ for (const client of connectedClients) {+ if (client.readyState === WebSocket.OPEN) {+ client.close();+ }+ }++ // Close the server+ await new Promise<void>((resolve) => {+ wss.close(() => resolve());+ });});describe("Connection Management", () => {@@ -40,16 +143,47 @@ describe("MacOSComputerInterface", () => {testParams.username,testParams.password,testParams.apiKey,- testParams.vmName+ testParams.vmName,+ false);await macosInterface.connect();// Verify the interface is connectedexpect(macosInterface.isConnected()).toBe(true);+ expect(connectedClients.length).toBe(1);await macosInterface.disconnect();});++ it("should handle connection without API key", async () => {+ // Create a separate server that doesn't check auth+ const noAuthWss = new WebSocketServer({ port: 0 });+ const noAuthPort = (noAuthWss.address() as { port: number }).port;++ noAuthWss.on("connection", (ws) => {+ ws.on("message", () => {+ ws.send(JSON.stringify({ success: true }));+ });+ });++ const macosInterface = new MacOSComputerInterface(+ `localhost:${noAuthPort}`,+ testParams.username,+ testParams.password,+ undefined,+ undefined,+ false+ );++ await macosInterface.connect();+ expect(macosInterface.isConnected()).toBe(true);++ await macosInterface.disconnect();+ await new Promise<void>((resolve) => {+ noAuthWss.close(() => resolve());+ });+ });});describe("Mouse Actions", () => {@@ -61,106 +195,109 @@ describe("MacOSComputerInterface", () => {testParams.username,testParams.password,testParams.apiKey,- testParams.vmName+ testParams.vmName,+ false);- // Remove initialize() call - connection happens on first command- await new Promise((resolve) => setTimeout(resolve, 100));+ await macosInterface.connect();});afterEach(async () => {- await macosInterface.disconnect();+ if (macosInterface) {+ await macosInterface.disconnect();+ }});it("should send mouse_down command", async () => {await macosInterface.mouseDown(100, 200, "left");-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({action: "mouse_down",x: 100,y: 200,- button: "left",+ button: "left"});});it("should send mouse_up command", async () => {await macosInterface.mouseUp(100, 200, "right");-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({action: "mouse_up",x: 100,y: 200,- button: "right",+ button: "right"});});it("should send left_click command", async () => {await macosInterface.leftClick(150, 250);-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({action: "left_click",x: 150,- y: 250,+ y: 250});});it("should send right_click command", async () => {- await macosInterface.rightClick(150, 250);-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({+ await macosInterface.rightClick(200, 300);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({action: "right_click",- x: 150,- y: 250,+ x: 200,+ y: 300});});it("should send double_click command", async () => {- await macosInterface.doubleClick(150, 250);-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({+ await macosInterface.doubleClick(250, 350);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({action: "double_click",- x: 150,- y: 250,+ x: 250,+ y: 350});});it("should send move_cursor command", async () => {await macosInterface.moveCursor(300, 400);-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({action: "move_cursor",x: 300,- y: 400,+ y: 400});});it("should send drag_to command", async () => {- await macosInterface.dragTo(500, 600, "left", 1.5);-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({+ await macosInterface.dragTo(400, 500, "left", 1.5);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({action: "drag_to",- x: 500,- y: 600,+ x: 400,+ y: 500,button: "left",- duration: 1.5,+ duration: 1.5});});- it("should send scroll command", async () => {- await macosInterface.scroll(0, 10);-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({- action: "scroll",- x: 0,- y: 10,- clicks: 5,+ it("should send drag command with path", async () => {+ const path: Array<[number, number]> = [[100, 100], [200, 200], [300, 300]];+ await macosInterface.drag(path, "middle", 2.0);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "drag",+ path: path,+ button: "middle",+ duration: 2.0});});});@@ -174,63 +311,118 @@ describe("MacOSComputerInterface", () => {testParams.username,testParams.password,testParams.apiKey,- testParams.vmName+ testParams.vmName,+ false);- // Remove initialize() call- await new Promise((resolve) => setTimeout(resolve, 100));+ await macosInterface.connect();});afterEach(async () => {- await macosInterface.disconnect();+ if (macosInterface) {+ await macosInterface.disconnect();+ }});it("should send key_down command", async () => {await macosInterface.keyDown("a");-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({action: "key_down",- key: "a",+ key: "a"});});it("should send key_up command", async () => {- await macosInterface.keyUp("a");-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({+ await macosInterface.keyUp("b");++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({action: "key_up",- key: "a",- });- });-- it("should send key_press command", async () => {- await macosInterface.keyDown("enter");-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({- action: "key_press",- key: "enter",+ key: "b"});});it("should send type_text command", async () => {await macosInterface.typeText("Hello, World!");-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({action: "type_text",- text: "Hello, World!",+ text: "Hello, World!"+ });+ });++ it("should send press_key command", async () => {+ await macosInterface.pressKey("enter");++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "press_key",+ key: "enter"});});it("should send hotkey command", async () => {await macosInterface.hotkey("cmd", "c");-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({action: "hotkey",- keys: ["cmd", "c"],+ keys: ["cmd", "c"]+ });+ });+ });++ describe("Scrolling Actions", () => {+ let macosInterface: MacOSComputerInterface;++ beforeEach(async () => {+ macosInterface = new MacOSComputerInterface(+ testParams.ipAddress,+ testParams.username,+ testParams.password,+ testParams.apiKey,+ testParams.vmName,+ false+ );+ await macosInterface.connect();+ });++ afterEach(async () => {+ if (macosInterface) {+ await macosInterface.disconnect();+ }+ });++ it("should send scroll command", async () => {+ await macosInterface.scroll(10, -5);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "scroll",+ x: 10,+ y: -5+ });+ });++ it("should send scroll_down command", async () => {+ await macosInterface.scrollDown(3);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "scroll_down",+ clicks: 3+ });+ });++ it("should send scroll_up command", async () => {+ await macosInterface.scrollUp(2);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "scroll_up",+ clicks: 2});});});@@ -244,62 +436,97 @@ describe("MacOSComputerInterface", () => {testParams.username,testParams.password,testParams.apiKey,- testParams.vmName+ testParams.vmName,+ false);- // Remove initialize() call- await new Promise((resolve) => setTimeout(resolve, 100));+ await macosInterface.connect();});afterEach(async () => {- await macosInterface.disconnect();+ if (macosInterface) {+ await macosInterface.disconnect();+ }});it("should get screenshot", async () => {const screenshot = await macosInterface.screenshot();-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({- action: "screenshot",++ expect(screenshot).toBeInstanceOf(Buffer);+ expect(screenshot.toString()).toBe("fake-screenshot-data");++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "screenshot"});- expect(screenshot).toBe("base64encodedimage");});it("should get screen size", async () => {- const screenSize = await macosInterface.getScreenSize();-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({- action: "get_screen_size",+ const size = await macosInterface.getScreenSize();++ expect(size).toEqual({ width: 1920, height: 1080 });++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "get_screen_size"});- expect(screenSize).toEqual({ width: 1920, height: 1080 });});it("should get cursor position", async () => {const position = await macosInterface.getCursorPosition();-- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({- action: "get_cursor_position",- });+expect(position).toEqual({ x: 100, y: 200 });++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "get_cursor_position"+ });});+ });- it("should get accessibility tree", async () => {- const tree = await macosInterface.getAccessibilityTree();+ describe("Clipboard Actions", () => {+ let macosInterface: MacOSComputerInterface;++ beforeEach(async () => {+ macosInterface = new MacOSComputerInterface(+ testParams.ipAddress,+ testParams.username,+ testParams.password,+ testParams.apiKey,+ testParams.vmName,+ false+ );+ await macosInterface.connect();+ });- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({- action: "get_accessibility_tree",+ afterEach(async () => {+ if (macosInterface) {+ await macosInterface.disconnect();+ }+ });++ it("should copy to clipboard", async () => {+ const text = await macosInterface.copyToClipboard();++ expect(text).toBe("clipboard content");++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "copy_to_clipboard"});- expect(tree).toEqual({- role: "window",- title: "Test Window",- children: [],+ });++ it("should set clipboard", async () => {+ await macosInterface.setClipboard("new clipboard text");++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "set_clipboard",+ text: "new clipboard text"});});});- describe("System Actions", () => {+ describe("File System Actions", () => {let macosInterface: MacOSComputerInterface;beforeEach(async () => {@@ -308,28 +535,208 @@ describe("MacOSComputerInterface", () => {testParams.username,testParams.password,testParams.apiKey,- testParams.vmName+ testParams.vmName,+ false);- // Remove initialize() call- await new Promise((resolve) => setTimeout(resolve, 100));+ await macosInterface.connect();});afterEach(async () => {- await macosInterface.disconnect();+ if (macosInterface) {+ await macosInterface.disconnect();+ }+ });++ it("should check file exists", async () => {+ const exists = await macosInterface.fileExists("/path/to/file");++ expect(exists).toBe(true);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "file_exists",+ path: "/path/to/file"+ });});- it("should run command", async () => {- const result = await macosInterface.runCommand("ls -la");+ it("should check directory exists", async () => {+ const exists = await macosInterface.directoryExists("/path/to/dir");++ expect(exists).toBe(true);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "directory_exists",+ path: "/path/to/dir"+ });+ });++ it("should list directory", async () => {+ const files = await macosInterface.listDir("/path/to/dir");++ expect(files).toEqual(["file1.txt", "file2.txt"]);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "list_dir",+ path: "/path/to/dir"+ });+ });++ it("should read text file", async () => {+ const content = await macosInterface.readText("/path/to/file.txt");++ expect(content).toBe("file content");++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "read_text",+ path: "/path/to/file.txt"+ });+ });++ it("should write text file", async () => {+ await macosInterface.writeText("/path/to/file.txt", "new content");++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "write_text",+ path: "/path/to/file.txt",+ content: "new content"+ });+ });++ it("should read binary file", async () => {+ const content = await macosInterface.readBytes("/path/to/file.bin");++ expect(content).toBeInstanceOf(Buffer);+ expect(content.toString()).toBe("binary content");++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "read_bytes",+ path: "/path/to/file.bin"+ });+ });++ it("should write binary file", async () => {+ const buffer = Buffer.from("binary data");+ await macosInterface.writeBytes("/path/to/file.bin", buffer);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "write_bytes",+ path: "/path/to/file.bin",+ content: buffer.toString("base64")+ });+ });- expect(receivedMessages).toHaveLength(1);- expect(receivedMessages[0]).toEqual({+ it("should delete file", async () => {+ await macosInterface.deleteFile("/path/to/file");++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "delete_file",+ path: "/path/to/file"+ });+ });++ it("should create directory", async () => {+ await macosInterface.createDir("/path/to/new/dir");++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "create_dir",+ path: "/path/to/new/dir"+ });+ });++ it("should delete directory", async () => {+ await macosInterface.deleteDir("/path/to/dir");++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "delete_dir",+ path: "/path/to/dir"+ });+ });++ it("should run command", async () => {+ const [stdout, stderr] = await macosInterface.runCommand("ls -la");++ expect(stdout).toBe("command output");+ expect(stderr).toBe("");++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({action: "run_command",- command: "ls -la",+ command: "ls -la"+ });+ });+ });++ describe("Accessibility Actions", () => {+ let macosInterface: MacOSComputerInterface;++ beforeEach(async () => {+ macosInterface = new MacOSComputerInterface(+ testParams.ipAddress,+ testParams.username,+ testParams.password,+ testParams.apiKey,+ testParams.vmName,+ false+ );+ await macosInterface.connect();+ });++ afterEach(async () => {+ if (macosInterface) {+ await macosInterface.disconnect();+ }+ });++ it("should get accessibility tree", async () => {+ const tree = await macosInterface.getAccessibilityTree();++ expect(tree).toEqual({+ role: "window",+ title: "Test Window",+ bounds: { x: 0, y: 0, width: 1920, height: 1080 },+ children: []});- expect(result).toEqual({- stdout: "command output",- stderr: "",- returncode: 0,++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "get_accessibility_tree"+ });+ });++ it("should convert to screen coordinates", async () => {+ const [x, y] = await macosInterface.toScreenCoordinates(100, 200);++ expect(x).toBe(100);+ expect(y).toBe(200);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "to_screen_coordinates",+ x: 100,+ y: 200+ });+ });++ it("should convert to screenshot coordinates", async () => {+ const [x, y] = await macosInterface.toScreenshotCoordinates(300, 400);++ expect(x).toBe(300);+ expect(y).toBe(400);++ const lastMessage = receivedMessages[receivedMessages.length - 1];+ expect(lastMessage).toEqual({+ action: "to_screenshot_coordinates",+ x: 300,+ y: 400});});});@@ -338,107 +745,89 @@ describe("MacOSComputerInterface", () => {it("should handle WebSocket connection errors", async () => {// Use a valid but unreachable IP to avoid DNS errorsconst macosInterface = new MacOSComputerInterface(- testParams.ipAddress,+ "localhost:9999",testParams.username,testParams.password,testParams.apiKey,- testParams.vmName+ testParams.vmName,+ false);- // Try to send a command - should fail with connection error- await expect(macosInterface.screenshot()).rejects.toThrow();-- await macosInterface.disconnect();+ // Connection should fail+ await expect(macosInterface.connect()).rejects.toThrow();});- it("should handle server error responses", async () => {- // Override the handler to send error response- // server.use(- // chat.addEventListener("connection", ({ client, server }) => {- // client.addEventListener("message", () => {- // server.send(- // JSON.stringify({- // success: false,- // error: "Command failed",- // })- // );- // });- // })- // );+ it("should handle command errors", async () => {+ // Create a server that returns errors+ const errorWss = new WebSocketServer({ port: 0 });+ const errorPort = (errorWss.address() as { port: number }).port;++ errorWss.on("connection", (ws) => {+ ws.on("message", () => {+ ws.send(JSON.stringify({ error: "Command failed" }));+ });+ });const macosInterface = new MacOSComputerInterface(- testParams.ipAddress,+ `localhost:${errorPort}`,testParams.username,testParams.password,testParams.apiKey,- testParams.vmName+ testParams.vmName,+ false);- // Remove initialize() call- await new Promise((resolve) => setTimeout(resolve, 100));- await expect(macosInterface.screenshot()).rejects.toThrow(- "Command failed"- );+ await macosInterface.connect();++ // Command should throw error+ await expect(macosInterface.leftClick(100, 100)).rejects.toThrow("Command failed");await macosInterface.disconnect();+ await new Promise<void>((resolve) => {+ errorWss.close(() => resolve());+ });});- it("should handle closed connection", async () => {+ it("should handle disconnection gracefully", async () => {const macosInterface = new MacOSComputerInterface(testParams.ipAddress,testParams.username,testParams.password,testParams.apiKey,- testParams.vmName+ testParams.vmName,+ false);- // Send a command to trigger connection- await macosInterface.screenshot();+ await macosInterface.connect();+ expect(macosInterface.isConnected()).toBe(true);- // Close the interface- await macosInterface.disconnect();+ // Disconnect+ macosInterface.disconnect();+ expect(macosInterface.isConnected()).toBe(false);- // Try to use after closing- await expect(macosInterface.screenshot()).rejects.toThrow(- "Interface is closed"- );- });- });+ // Should reconnect automatically on next command+ await macosInterface.leftClick(100, 100);+ expect(macosInterface.isConnected()).toBe(true);- describe("Command Locking", () => {- let macosInterface: MacOSComputerInterface;+ await macosInterface.disconnect();+ });- beforeEach(async () => {- macosInterface = new MacOSComputerInterface(+ it("should handle force close", async () => {+ const macosInterface = new MacOSComputerInterface(testParams.ipAddress,testParams.username,testParams.password,testParams.apiKey,- testParams.vmName+ testParams.vmName,+ false);- // Remove initialize() call- await new Promise((resolve) => setTimeout(resolve, 100));- });-- afterEach(async () => {- await macosInterface.disconnect();- });- it("should serialize commands", async () => {- // Send multiple commands simultaneously- const promises = [- macosInterface.leftClick(100, 100),- macosInterface.rightClick(200, 200),- macosInterface.typeText("test"),- ];-- await Promise.all(promises);+ await macosInterface.connect();+ expect(macosInterface.isConnected()).toBe(true);- // Commands should be sent in order- expect(receivedMessages).toHaveLength(3);- expect(receivedMessages[0].action).toBe("left_click");- expect(receivedMessages[1].action).toBe("right_click");- expect(receivedMessages[2].action).toBe("type_text");+ // Force close+ macosInterface.forceClose();+ expect(macosInterface.isConnected()).toBe(false);});});});diff --git a/libs/computer/typescript/tests/setup.ts b/libs/computer/typescript/tests/setup.tsindex 42b09201f1..7dd1ebc769 100644--- a/libs/computer/typescript/tests/setup.ts+++ b/libs/computer/typescript/tests/setup.ts@@ -1,26 +1,7 @@import { afterAll, afterEach, beforeAll } from "vitest";-import { setupServer } from "msw/node";-import { ws } from "msw";-const chat = ws.link("wss://chat.example.com");+beforeAll(() => {});-const wsHandlers = [- chat.addEventListener("connection", ({ client }) => {- client.addEventListener("message", (event) => {- console.log("Received message from client:", event.data);- // Echo the received message back to the client- client.send(`Server received: ${event.data}`);- });- }),-];+afterAll(() => {});-const server = setupServer(...wsHandlers);--// Start server before all tests-beforeAll(() => server.listen({ onUnhandledRequest: "error" }));--// Close server after all tests-afterAll(() => server.close());--// Reset handlers after each test for test isolation-afterEach(() => server.resetHandlers());+afterEach(() => {});