Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Update tests for expected server responses
Morgan Dean8be7dafa3e7b
diff --git a/libs/computer/typescript/tests/interface/macos.test.ts b/libs/computer/typescript/tests/interface/macos.test.tsindex 78d2d9d3a5..68b03583d6 100644--- a/libs/computer/typescript/tests/interface/macos.test.ts+++ b/libs/computer/typescript/tests/interface/macos.test.ts@@ -56,54 +56,83 @@ describe("MacOSComputerInterface", () => {receivedMessages.push(message);// Send appropriate responses based on action- switch (message.action) {+ switch (message.command) {case "screenshot":ws.send(JSON.stringify({- data: Buffer.from("fake-screenshot-data").toString("base64")+ image_data: Buffer.from("fake-screenshot-data").toString("base64"),+ success: true}));break;case "get_screen_size":- ws.send(JSON.stringify({ data: { width: 1920, height: 1080 } }));+ ws.send(JSON.stringify({+ size: { width: 1920, height: 1080 },+ success: true+ }));break;case "get_cursor_position":- ws.send(JSON.stringify({ data: { x: 100, y: 200 } }));+ ws.send(JSON.stringify({+ position: { x: 100, y: 200 },+ success: true+ }));break;case "copy_to_clipboard":- ws.send(JSON.stringify({ data: "clipboard content" }));+ ws.send(JSON.stringify({+ content: "clipboard content",+ success: true+ }));break;case "file_exists":- ws.send(JSON.stringify({ data: true }));+ ws.send(JSON.stringify({+ exists: true,+ success: true+ }));break;case "directory_exists":- ws.send(JSON.stringify({ data: true }));+ ws.send(JSON.stringify({+ exists: true,+ success: true+ }));break;case "list_dir":- ws.send(JSON.stringify({ data: ["file1.txt", "file2.txt"] }));+ ws.send(JSON.stringify({+ files: ["file1.txt", "file2.txt"],+ success: true+ }));break;case "read_text":- ws.send(JSON.stringify({ data: "file content" }));+ ws.send(JSON.stringify({+ content: "file content",+ success: true+ }));break;case "read_bytes":ws.send(JSON.stringify({- data: Buffer.from("binary content").toString("base64")+ content_b64: Buffer.from("binary content").toString("base64"),+ success: true}));break;case "run_command":- ws.send(JSON.stringify({ data: { stdout: "command output", stderr: "" } }));+ ws.send(JSON.stringify({+ stdout: "command output",+ stderr: "",+ success: true+ }));break;case "get_accessibility_tree":ws.send(JSON.stringify({- data: {- role: "window",- title: "Test Window",- bounds: { x: 0, y: 0, width: 1920, height: 1080 },- children: []- }+ role: "window",+ title: "Test Window",+ bounds: { x: 0, y: 0, width: 1920, height: 1080 },+ children: [],+ success: true}));break;case "to_screen_coordinates":case "to_screenshot_coordinates":- ws.send(JSON.stringify({ data: [message.x || 0, message.y || 0] }));+ ws.send(JSON.stringify({+ coordinates: [message.params?.x || 0, message.params?.y || 0],+ success: true+ }));break;default:// For all other actions, just send success@@ -212,10 +241,12 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "mouse_down",- x: 100,- y: 200,- button: "left"+ command: "mouse_down",+ params: {+ x: 100,+ y: 200,+ button: "left"+ }});});@@ -224,10 +255,12 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "mouse_up",- x: 100,- y: 200,- button: "right"+ command: "mouse_up",+ params: {+ x: 100,+ y: 200,+ button: "right"+ }});});@@ -236,9 +269,11 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "left_click",- x: 150,- y: 250+ command: "left_click",+ params: {+ x: 150,+ y: 250+ }});});@@ -247,9 +282,11 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "right_click",- x: 200,- y: 300+ command: "right_click",+ params: {+ x: 200,+ y: 300+ }});});@@ -258,9 +295,11 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "double_click",- x: 250,- y: 350+ command: "double_click",+ params: {+ x: 250,+ y: 350+ }});});@@ -269,9 +308,11 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "move_cursor",- x: 300,- y: 400+ command: "move_cursor",+ params: {+ x: 300,+ y: 400+ }});});@@ -280,11 +321,13 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "drag_to",- x: 400,- y: 500,- button: "left",- duration: 1.5+ command: "drag_to",+ params: {+ x: 400,+ y: 500,+ button: "left",+ duration: 1.5+ }});});@@ -294,10 +337,12 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "drag",- path: path,- button: "middle",- duration: 2.0+ command: "drag",+ params: {+ path: path,+ button: "middle",+ duration: 2.0+ }});});});@@ -328,8 +373,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "key_down",- key: "a"+ command: "key_down",+ params: {+ key: "a"+ }});});@@ -338,8 +385,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "key_up",- key: "b"+ command: "key_up",+ params: {+ key: "b"+ }});});@@ -348,8 +397,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "type_text",- text: "Hello, World!"+ command: "type_text",+ params: {+ text: "Hello, World!"+ }});});@@ -358,8 +409,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "press_key",- key: "enter"+ command: "press_key",+ params: {+ key: "enter"+ }});});@@ -368,8 +421,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "hotkey",- keys: ["cmd", "c"]+ command: "hotkey",+ params: {+ keys: ["cmd", "c"]+ }});});});@@ -400,9 +455,11 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "scroll",- x: 10,- y: -5+ command: "scroll",+ params: {+ x: 10,+ y: -5+ }});});@@ -411,8 +468,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "scroll_down",- clicks: 3+ command: "scroll_down",+ params: {+ clicks: 3+ }});});@@ -421,8 +480,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "scroll_up",- clicks: 2+ command: "scroll_up",+ params: {+ clicks: 2+ }});});});@@ -456,7 +517,8 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "screenshot"+ command: "screenshot",+ params: {}});});@@ -467,7 +529,8 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "get_screen_size"+ command: "get_screen_size",+ params: {}});});@@ -478,7 +541,8 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "get_cursor_position"+ command: "get_cursor_position",+ params: {}});});});@@ -511,7 +575,8 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "copy_to_clipboard"+ command: "copy_to_clipboard",+ params: {}});});@@ -520,8 +585,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "set_clipboard",- text: "new clipboard text"+ command: "set_clipboard",+ params: {+ text: "new clipboard text"+ }});});});@@ -554,8 +621,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "file_exists",- path: "/path/to/file"+ command: "file_exists",+ params: {+ path: "/path/to/file"+ }});});@@ -566,8 +635,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "directory_exists",- path: "/path/to/dir"+ command: "directory_exists",+ params: {+ path: "/path/to/dir"+ }});});@@ -578,8 +649,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "list_dir",- path: "/path/to/dir"+ command: "list_dir",+ params: {+ path: "/path/to/dir"+ }});});@@ -590,8 +663,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "read_text",- path: "/path/to/file.txt"+ command: "read_text",+ params: {+ path: "/path/to/file.txt"+ }});});@@ -600,9 +675,11 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "write_text",- path: "/path/to/file.txt",- content: "new content"+ command: "write_text",+ params: {+ path: "/path/to/file.txt",+ content: "new content"+ }});});@@ -614,8 +691,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "read_bytes",- path: "/path/to/file.bin"+ command: "read_bytes",+ params: {+ path: "/path/to/file.bin"+ }});});@@ -625,9 +704,11 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "write_bytes",- path: "/path/to/file.bin",- content: buffer.toString("base64")+ command: "write_bytes",+ params: {+ path: "/path/to/file.bin",+ content_b64: buffer.toString("base64")+ }});});@@ -636,8 +717,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "delete_file",- path: "/path/to/file"+ command: "delete_file",+ params: {+ path: "/path/to/file"+ }});});@@ -646,8 +729,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "create_dir",- path: "/path/to/new/dir"+ command: "create_dir",+ params: {+ path: "/path/to/new/dir"+ }});});@@ -656,8 +741,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "delete_dir",- path: "/path/to/dir"+ command: "delete_dir",+ params: {+ path: "/path/to/dir"+ }});});@@ -669,8 +756,10 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "run_command",- command: "ls -la"+ command: "run_command",+ params: {+ command: "ls -la"+ }});});});@@ -703,12 +792,14 @@ describe("MacOSComputerInterface", () => {role: "window",title: "Test Window",bounds: { x: 0, y: 0, width: 1920, height: 1080 },- children: []+ children: [],+ success: true});const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "get_accessibility_tree"+ command: "get_accessibility_tree",+ params: {}});});@@ -720,9 +811,11 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "to_screen_coordinates",- x: 100,- y: 200+ command: "to_screen_coordinates",+ params: {+ x: 100,+ y: 200+ }});});@@ -734,9 +827,11 @@ describe("MacOSComputerInterface", () => {const lastMessage = receivedMessages[receivedMessages.length - 1];expect(lastMessage).toEqual({- action: "to_screenshot_coordinates",- x: 300,- y: 400+ command: "to_screenshot_coordinates",+ params: {+ x: 300,+ y: 400+ }});});});@@ -764,7 +859,7 @@ describe("MacOSComputerInterface", () => {errorWss.on("connection", (ws) => {ws.on("message", () => {- ws.send(JSON.stringify({ error: "Command failed" }));+ ws.send(JSON.stringify({ error: "Command failed", success: false }));});});