Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Fix issues with connecting to cloud vms, update tests
Morgan Dean3e6ef652459a
diff --git a/libs/computer/typescript/src/computer/providers/cloud.ts b/libs/computer/typescript/src/computer/providers/cloud.tsindex 7e8df49570..a1e458dc4c 100644--- a/libs/computer/typescript/src/computer/providers/cloud.ts+++ b/libs/computer/typescript/src/computer/providers/cloud.ts@@ -23,8 +23,7 @@ export class CloudComputer extends BaseComputer {}get ip() {- return "192.168.64.9";- //return `${this.name}.containers.cloud.trycua.com`;+ return `${this.name}.containers.cloud.trycua.com`;}/**@@ -36,8 +35,6 @@ export class CloudComputer extends BaseComputer {return;}- logger.info("Starting cloud computer...");-try {// For cloud provider, the VM is already running, we just need to connectconst ipAddress = this.ip;@@ -67,7 +64,7 @@ export class CloudComputer extends BaseComputer {* Stop the cloud computer (disconnect interface)*/async stop(): Promise<void> {- logger.info("Stopping cloud computer...");+ logger.info("Disconnecting from cloud computer...");if (this.iface) {this.iface.disconnect();@@ -75,7 +72,7 @@ export class CloudComputer extends BaseComputer {}this.initialized = false;- logger.info("Cloud computer stopped");+ logger.info("Disconnected from cloud computer");}/**diff --git a/libs/computer/typescript/src/interface/base.ts b/libs/computer/typescript/src/interface/base.tsindex 6f012fe688..a69002d45f 100644--- a/libs/computer/typescript/src/interface/base.ts+++ b/libs/computer/typescript/src/interface/base.ts@@ -39,7 +39,6 @@ export abstract class BaseComputerInterface {protected ws: WebSocket;protected apiKey?: string;protected vmName?: string;- protected secure?: boolean;protected logger = pino({ name: "interface-base" });@@ -48,15 +47,13 @@ export abstract class BaseComputerInterface {username = "lume",password = "lume",apiKey?: string,- vmName?: string,- secure?: boolean+ vmName?: string) {this.ipAddress = ipAddress;this.username = username;this.password = password;this.apiKey = apiKey;this.vmName = vmName;- this.secure = secure;// Initialize WebSocket with headers if neededconst headers: { [key: string]: string } = {};@@ -74,7 +71,7 @@ export abstract class BaseComputerInterface {* Subclasses can override this to customize the URI.*/protected get wsUri(): string {- const protocol = this.secure ? "wss" : "ws";+ const protocol = this.apiKey ? "wss" : "ws";// Check if ipAddress already includes a portif (this.ipAddress.includes(":")) {@@ -82,7 +79,7 @@ export abstract class BaseComputerInterface {}// Otherwise, append the default port- const port = this.secure ? "8443" : "8000";+ const port = this.apiKey ? "8443" : "8000";return `${protocol}://${this.ipAddress}:${port}/ws`;}@@ -99,6 +96,7 @@ export abstract class BaseComputerInterface {await this.connect();return;} catch (error) {+ console.log(error);// Wait a bit before retryingthis.logger.error(`Error connecting to websocket: ${JSON.stringify(error)}`@@ -115,6 +113,42 @@ export abstract class BaseComputerInterface {*/public async connect(): Promise<void> {if (this.ws.readyState === WebSocket.OPEN) {+ // send authentication message if needed+ if (this.apiKey && this.vmName) {+ this.logger.info("Performing authentication handshake...");+ const authMessage = {+ command: "authenticate",+ params: {+ api_key: this.apiKey,+ container_name: this.vmName,+ },+ };++ return new Promise<void>((resolve, reject) => {+ const authHandler = (data: WebSocket.RawData) => {+ try {+ const authResult = JSON.parse(data.toString());+ if (!authResult.success) {+ const errorMsg = authResult.error || "Authentication failed";+ this.logger.error(`Authentication failed: ${errorMsg}`);+ this.ws.close();+ reject(new Error(`Authentication failed: ${errorMsg}`));+ } else {+ this.logger.info("Authentication successful");+ this.ws.off("message", authHandler);+ resolve();+ }+ } catch (error) {+ this.ws.off("message", authHandler);+ reject(error);+ }+ };++ this.ws.on("message", authHandler);+ this.ws.send(JSON.stringify(authMessage));+ });+ }+return;}diff --git a/libs/computer/typescript/tests/interface/macos.test.ts b/libs/computer/typescript/tests/interface/macos.test.tsindex 68b03583d6..72ee07ae2d 100644--- a/libs/computer/typescript/tests/interface/macos.test.ts+++ b/libs/computer/typescript/tests/interface/macos.test.ts@@ -8,7 +8,7 @@ describe("MacOSComputerInterface", () => {ipAddress: "localhost",username: "testuser",password: "testpass",- apiKey: "test-api-key",+ // apiKey: "test-api-key", No API Key for local testingvmName: "test-vm",};@@ -37,18 +37,9 @@ describe("MacOSComputerInterface", () => {testParams.ipAddress = `localhost:${serverPort}`;// Handle WebSocket connections- wss.on("connection", (ws, req) => {+ wss.on("connection", (ws) => {connectedClients.push(ws);- // Verify authentication headers- const apiKey = req.headers["x-api-key"];- const vmName = req.headers["x-vm-name"];-- if (apiKey !== testParams.apiKey || vmName !== testParams.vmName) {- ws.close(1008, "Unauthorized");- return;- }-// Handle incoming messagesws.on("message", (data) => {try {@@ -58,81 +49,107 @@ describe("MacOSComputerInterface", () => {// Send appropriate responses based on actionswitch (message.command) {case "screenshot":- ws.send(JSON.stringify({- image_data: Buffer.from("fake-screenshot-data").toString("base64"),- success: true- }));+ ws.send(+ JSON.stringify({+ image_data: Buffer.from("fake-screenshot-data").toString(+ "base64"+ ),+ success: true,+ })+ );break;case "get_screen_size":- ws.send(JSON.stringify({- size: { width: 1920, height: 1080 },- success: true- }));+ ws.send(+ JSON.stringify({+ size: { width: 1920, height: 1080 },+ success: true,+ })+ );break;case "get_cursor_position":- ws.send(JSON.stringify({- position: { x: 100, y: 200 },- success: true- }));+ ws.send(+ JSON.stringify({+ position: { x: 100, y: 200 },+ success: true,+ })+ );break;case "copy_to_clipboard":- ws.send(JSON.stringify({- content: "clipboard content",- success: true- }));+ ws.send(+ JSON.stringify({+ content: "clipboard content",+ success: true,+ })+ );break;case "file_exists":- ws.send(JSON.stringify({- exists: true,- success: true- }));+ ws.send(+ JSON.stringify({+ exists: true,+ success: true,+ })+ );break;case "directory_exists":- ws.send(JSON.stringify({- exists: true,- success: true- }));+ ws.send(+ JSON.stringify({+ exists: true,+ success: true,+ })+ );break;case "list_dir":- ws.send(JSON.stringify({- files: ["file1.txt", "file2.txt"],- success: true- }));+ ws.send(+ JSON.stringify({+ files: ["file1.txt", "file2.txt"],+ success: true,+ })+ );break;case "read_text":- ws.send(JSON.stringify({- content: "file content",- success: true- }));+ ws.send(+ JSON.stringify({+ content: "file content",+ success: true,+ })+ );break;case "read_bytes":- ws.send(JSON.stringify({- content_b64: Buffer.from("binary content").toString("base64"),- success: true- }));+ ws.send(+ JSON.stringify({+ content_b64: Buffer.from("binary content").toString("base64"),+ success: true,+ })+ );break;case "run_command":- ws.send(JSON.stringify({- stdout: "command output",- stderr: "",- success: true- }));+ ws.send(+ JSON.stringify({+ stdout: "command output",+ stderr: "",+ success: true,+ })+ );break;case "get_accessibility_tree":- ws.send(JSON.stringify({- role: "window",- title: "Test Window",- bounds: { x: 0, y: 0, width: 1920, height: 1080 },- children: [],- success: true- }));+ ws.send(+ JSON.stringify({+ role: "window",+ title: "Test Window",+ bounds: { x: 0, y: 0, width: 1920, height: 1080 },+ children: [],+ success: true,+ })+ );break;case "to_screen_coordinates":case "to_screenshot_coordinates":- ws.send(JSON.stringify({- coordinates: [message.params?.x || 0, message.params?.y || 0],- success: true- }));+ ws.send(+ JSON.stringify({+ coordinates: [message.params?.x || 0, message.params?.y || 0],+ success: true,+ })+ );break;default:// For all other actions, just send success@@ -171,9 +188,8 @@ describe("MacOSComputerInterface", () => {testParams.ipAddress,testParams.username,testParams.password,- testParams.apiKey,- testParams.vmName,- false+ undefined,+ testParams.vmName);await macosInterface.connect();@@ -201,8 +217,7 @@ describe("MacOSComputerInterface", () => {testParams.username,testParams.password,undefined,- undefined,- false+ undefined);await macosInterface.connect();@@ -223,9 +238,8 @@ describe("MacOSComputerInterface", () => {testParams.ipAddress,testParams.username,testParams.password,- testParams.apiKey,- testParams.vmName,- false+ undefined,+ testParams.vmName);await macosInterface.connect();});@@ -238,87 +252,87 @@ describe("MacOSComputerInterface", () => {it("should send mouse_down command", async () => {await macosInterface.mouseDown(100, 200, "left");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "mouse_down",params: {x: 100,y: 200,- button: "left"- }+ button: "left",+ },});});it("should send mouse_up command", async () => {await macosInterface.mouseUp(100, 200, "right");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "mouse_up",params: {x: 100,y: 200,- button: "right"- }+ button: "right",+ },});});it("should send left_click command", async () => {await macosInterface.leftClick(150, 250);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "left_click",params: {x: 150,- y: 250- }+ y: 250,+ },});});it("should send right_click command", async () => {await macosInterface.rightClick(200, 300);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "right_click",params: {x: 200,- y: 300- }+ y: 300,+ },});});it("should send double_click command", async () => {await macosInterface.doubleClick(250, 350);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "double_click",params: {x: 250,- y: 350- }+ y: 350,+ },});});it("should send move_cursor command", async () => {await macosInterface.moveCursor(300, 400);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "move_cursor",params: {x: 300,- y: 400- }+ y: 400,+ },});});it("should send drag_to command", async () => {await macosInterface.dragTo(400, 500, "left", 1.5);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "drag_to",@@ -326,23 +340,27 @@ describe("MacOSComputerInterface", () => {x: 400,y: 500,button: "left",- duration: 1.5- }+ duration: 1.5,+ },});});it("should send drag command with path", async () => {- const path: Array<[number, number]> = [[100, 100], [200, 200], [300, 300]];+ const path: Array<[number, number]> = [+ [100, 100],+ [200, 200],+ [300, 300],+ ];await macosInterface.drag(path, "middle", 2.0);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "drag",params: {path: path,button: "middle",- duration: 2.0- }+ duration: 2.0,+ },});});});@@ -355,9 +373,8 @@ describe("MacOSComputerInterface", () => {testParams.ipAddress,testParams.username,testParams.password,- testParams.apiKey,- testParams.vmName,- false+ undefined,+ testParams.vmName);await macosInterface.connect();});@@ -370,61 +387,61 @@ describe("MacOSComputerInterface", () => {it("should send key_down command", async () => {await macosInterface.keyDown("a");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "key_down",params: {- key: "a"- }+ key: "a",+ },});});it("should send key_up command", async () => {await macosInterface.keyUp("b");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "key_up",params: {- key: "b"- }+ key: "b",+ },});});it("should send type_text command", async () => {await macosInterface.typeText("Hello, World!");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "type_text",params: {- text: "Hello, World!"- }+ text: "Hello, World!",+ },});});it("should send press_key command", async () => {await macosInterface.pressKey("enter");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "press_key",params: {- key: "enter"- }+ key: "enter",+ },});});it("should send hotkey command", async () => {await macosInterface.hotkey("cmd", "c");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "hotkey",params: {- keys: ["cmd", "c"]- }+ keys: ["cmd", "c"],+ },});});});@@ -437,9 +454,8 @@ describe("MacOSComputerInterface", () => {testParams.ipAddress,testParams.username,testParams.password,- testParams.apiKey,- testParams.vmName,- false+ undefined,+ testParams.vmName);await macosInterface.connect();});@@ -452,38 +468,38 @@ describe("MacOSComputerInterface", () => {it("should send scroll command", async () => {await macosInterface.scroll(10, -5);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "scroll",params: {x: 10,- y: -5- }+ y: -5,+ },});});it("should send scroll_down command", async () => {await macosInterface.scrollDown(3);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "scroll_down",params: {- clicks: 3- }+ clicks: 3,+ },});});it("should send scroll_up command", async () => {await macosInterface.scrollUp(2);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "scroll_up",params: {- clicks: 2- }+ clicks: 2,+ },});});});@@ -496,9 +512,8 @@ describe("MacOSComputerInterface", () => {testParams.ipAddress,testParams.username,testParams.password,- testParams.apiKey,- testParams.vmName,- false+ undefined,+ testParams.vmName);await macosInterface.connect();});@@ -511,38 +526,38 @@ describe("MacOSComputerInterface", () => {it("should get screenshot", async () => {const screenshot = await macosInterface.screenshot();-+expect(screenshot).toBeInstanceOf(Buffer);expect(screenshot.toString()).toBe("fake-screenshot-data");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "screenshot",- params: {}+ params: {},});});it("should get screen size", async () => {const size = await macosInterface.getScreenSize();-+expect(size).toEqual({ width: 1920, height: 1080 });-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "get_screen_size",- params: {}+ params: {},});});it("should get cursor position", async () => {const position = await macosInterface.getCursorPosition();-+expect(position).toEqual({ x: 100, y: 200 });-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "get_cursor_position",- params: {}+ params: {},});});});@@ -555,9 +570,8 @@ describe("MacOSComputerInterface", () => {testParams.ipAddress,testParams.username,testParams.password,- testParams.apiKey,- testParams.vmName,- false+ undefined,+ testParams.vmName);await macosInterface.connect();});@@ -570,25 +584,25 @@ describe("MacOSComputerInterface", () => {it("should copy to clipboard", async () => {const text = await macosInterface.copyToClipboard();-+expect(text).toBe("clipboard content");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "copy_to_clipboard",- params: {}+ params: {},});});it("should set clipboard", async () => {await macosInterface.setClipboard("new clipboard text");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "set_clipboard",params: {- text: "new clipboard text"- }+ text: "new clipboard text",+ },});});});@@ -601,9 +615,8 @@ describe("MacOSComputerInterface", () => {testParams.ipAddress,testParams.username,testParams.password,- testParams.apiKey,- testParams.vmName,- false+ undefined,+ testParams.vmName);await macosInterface.connect();});@@ -616,150 +629,150 @@ describe("MacOSComputerInterface", () => {it("should check file exists", async () => {const exists = await macosInterface.fileExists("/path/to/file");-+expect(exists).toBe(true);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "file_exists",params: {- path: "/path/to/file"- }+ path: "/path/to/file",+ },});});it("should check directory exists", async () => {const exists = await macosInterface.directoryExists("/path/to/dir");-+expect(exists).toBe(true);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "directory_exists",params: {- path: "/path/to/dir"- }+ path: "/path/to/dir",+ },});});it("should list directory", async () => {const files = await macosInterface.listDir("/path/to/dir");-+expect(files).toEqual(["file1.txt", "file2.txt"]);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "list_dir",params: {- path: "/path/to/dir"- }+ path: "/path/to/dir",+ },});});it("should read text file", async () => {const content = await macosInterface.readText("/path/to/file.txt");-+expect(content).toBe("file content");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "read_text",params: {- path: "/path/to/file.txt"- }+ path: "/path/to/file.txt",+ },});});it("should write text file", async () => {await macosInterface.writeText("/path/to/file.txt", "new content");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "write_text",params: {path: "/path/to/file.txt",- content: "new content"- }+ content: "new content",+ },});});it("should read binary file", async () => {const content = await macosInterface.readBytes("/path/to/file.bin");-+expect(content).toBeInstanceOf(Buffer);expect(content.toString()).toBe("binary content");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "read_bytes",params: {- path: "/path/to/file.bin"- }+ path: "/path/to/file.bin",+ },});});it("should write binary file", async () => {const buffer = Buffer.from("binary data");await macosInterface.writeBytes("/path/to/file.bin", buffer);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "write_bytes",params: {path: "/path/to/file.bin",- content_b64: buffer.toString("base64")- }+ content_b64: buffer.toString("base64"),+ },});});it("should delete file", async () => {await macosInterface.deleteFile("/path/to/file");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "delete_file",params: {- path: "/path/to/file"- }+ path: "/path/to/file",+ },});});it("should create directory", async () => {await macosInterface.createDir("/path/to/new/dir");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "create_dir",params: {- path: "/path/to/new/dir"- }+ path: "/path/to/new/dir",+ },});});it("should delete directory", async () => {await macosInterface.deleteDir("/path/to/dir");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "delete_dir",params: {- path: "/path/to/dir"- }+ path: "/path/to/dir",+ },});});it("should run command", async () => {const [stdout, stderr] = await macosInterface.runCommand("ls -la");-+expect(stdout).toBe("command output");expect(stderr).toBe("");-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "run_command",params: {- command: "ls -la"- }+ command: "ls -la",+ },});});});@@ -772,9 +785,8 @@ describe("MacOSComputerInterface", () => {testParams.ipAddress,testParams.username,testParams.password,- testParams.apiKey,- testParams.vmName,- false+ undefined,+ testParams.vmName);await macosInterface.connect();});@@ -787,51 +799,51 @@ describe("MacOSComputerInterface", () => {it("should get accessibility tree", async () => {const tree = await macosInterface.getAccessibilityTree();-+expect(tree).toEqual({role: "window",title: "Test Window",bounds: { x: 0, y: 0, width: 1920, height: 1080 },children: [],- success: true+ success: true,});-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "get_accessibility_tree",- params: {}+ params: {},});});it("should convert to screen coordinates", async () => {const [x, y] = await macosInterface.toScreenCoordinates(100, 200);-+expect(x).toBe(100);expect(y).toBe(200);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "to_screen_coordinates",params: {x: 100,- y: 200- }+ y: 200,+ },});});it("should convert to screenshot coordinates", async () => {const [x, y] = await macosInterface.toScreenshotCoordinates(300, 400);-+expect(x).toBe(300);expect(y).toBe(400);-+const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({command: "to_screenshot_coordinates",params: {x: 300,- y: 400- }+ y: 400,+ },});});});@@ -843,9 +855,8 @@ describe("MacOSComputerInterface", () => {"localhost:9999",testParams.username,testParams.password,- testParams.apiKey,- testParams.vmName,- false+ undefined,+ testParams.vmName);// Connection should fail@@ -867,15 +878,16 @@ describe("MacOSComputerInterface", () => {`localhost:${errorPort}`,testParams.username,testParams.password,- testParams.apiKey,- testParams.vmName,- false+ undefined,+ testParams.vmName);await macosInterface.connect();// Command should throw error- await expect(macosInterface.leftClick(100, 100)).rejects.toThrow("Command failed");+ await expect(macosInterface.leftClick(100, 100)).rejects.toThrow(+ "Command failed"+ );await macosInterface.disconnect();await new Promise<void>((resolve) => {@@ -888,9 +900,8 @@ describe("MacOSComputerInterface", () => {testParams.ipAddress,testParams.username,testParams.password,- testParams.apiKey,- testParams.vmName,- false+ undefined,+ testParams.vmName);await macosInterface.connect();@@ -912,9 +923,8 @@ describe("MacOSComputerInterface", () => {testParams.ipAddress,testParams.username,testParams.password,- testParams.apiKey,- testParams.vmName,- false+ undefined,+ testParams.vmName);await macosInterface.connect();