Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Added computer to client
Dillon DuPont137b336e317f
diff --git a/libs/computer/computer/interface/factory.py b/libs/computer/computer/interface/factory.pyindex 949972c495..3647400e96 100644--- a/libs/computer/computer/interface/factory.py+++ b/libs/computer/computer/interface/factory.py@@ -8,7 +8,7 @@ class InterfaceFactory:@staticmethoddef create_interface_for_os(- os: Literal['macos', 'linux'],+ os: Literal['macos', 'linux', 'windows'],ip_address: str,api_key: Optional[str] = None,vm_name: Optional[str] = None@@ -16,7 +16,7 @@ class InterfaceFactory:"""Create an interface for the specified OS.Args:- os: Operating system type ('macos' or 'linux')+ os: Operating system type ('macos', 'linux', or 'windows')ip_address: IP address of the computer to controlapi_key: Optional API key for cloud authenticationvm_name: Optional VM name for cloud authentication@@ -30,10 +30,13 @@ class InterfaceFactory:# Import implementations here to avoid circular importsfrom .macos import MacOSComputerInterfacefrom .linux import LinuxComputerInterface+ from .windows import WindowsComputerInterfaceif os == 'macos':return MacOSComputerInterface(ip_address, api_key=api_key, vm_name=vm_name)elif os == 'linux':return LinuxComputerInterface(ip_address, api_key=api_key, vm_name=vm_name)+ elif os == 'windows':+ return WindowsComputerInterface(ip_address, api_key=api_key, vm_name=vm_name)else:- raise ValueError(f"Unsupported OS type: {os}")\ No newline at end of file+ raise ValueError(f"Unsupported OS type: {os}")diff --git a/libs/computer/computer/interface/windows.py b/libs/computer/computer/interface/windows.pynew file mode 100644index 0000000000..6acce1c1ed--- /dev/null+++ b/libs/computer/computer/interface/windows.py@@ -0,0 +1,687 @@+import asyncio+import json+import time+from typing import Any, Dict, List, Optional, Tuple+from PIL import Image++import websockets++from ..logger import Logger, LogLevel+from .base import BaseComputerInterface+from ..utils import decode_base64_image, encode_base64_image, bytes_to_image, draw_box, resize_image+from .models import Key, KeyType, MouseButton+++class WindowsComputerInterface(BaseComputerInterface):+ """Interface for Windows."""++ def __init__(self, ip_address: str, username: str = "lume", password: str = "lume", api_key: Optional[str] = None, vm_name: Optional[str] = None):+ super().__init__(ip_address, username, password, api_key, vm_name)+ self._ws = None+ self._reconnect_task = None+ self._closed = False+ self._last_ping = 0+ self._ping_interval = 5 # Send ping every 5 seconds+ self._ping_timeout = 120 # Wait 120 seconds for pong response+ self._reconnect_delay = 1 # Start with 1 second delay+ self._max_reconnect_delay = 30 # Maximum delay between reconnection attempts+ self._log_connection_attempts = True # Flag to control connection attempt logging+ self._authenticated = False # Track authentication status+ self._command_lock = asyncio.Lock() # Lock to ensure only one command at a time++ # Set logger name for Windows interface+ self.logger = Logger("cua.interface.windows", LogLevel.NORMAL)++ @property+ def ws_uri(self) -> str:+ """Get the WebSocket URI using the current IP address.++ Returns:+ WebSocket URI for the Computer API Server+ """+ protocol = "wss" if self.api_key else "ws"+ port = "8443" if self.api_key else "8000"+ return f"{protocol}://{self.ip_address}:{port}/ws"++ async def _keep_alive(self):+ """Keep the WebSocket connection alive with automatic reconnection."""+ retry_count = 0+ max_log_attempts = 1 # Only log the first attempt at INFO level+ log_interval = 500 # Then log every 500th attempt (significantly increased from 30)+ last_warning_time = 0+ min_warning_interval = 30 # Minimum seconds between connection lost warnings+ min_retry_delay = 0.5 # Minimum delay between connection attempts (500ms)++ while not self._closed:+ try:+ if self._ws is None or (+ self._ws and self._ws.state == websockets.protocol.State.CLOSED+ ):+ try:+ retry_count += 1++ # Add a minimum delay between connection attempts to avoid flooding+ if retry_count > 1:+ await asyncio.sleep(min_retry_delay)++ # Only log the first attempt at INFO level, then every Nth attempt+ if retry_count == 1:+ self.logger.info(f"Attempting WebSocket connection to {self.ws_uri}")+ elif retry_count % log_interval == 0:+ self.logger.info(+ f"Still attempting WebSocket connection (attempt {retry_count})..."+ )+ else:+ # All other attempts are logged at DEBUG level+ self.logger.debug(+ f"Attempting WebSocket connection to {self.ws_uri} (attempt {retry_count})"+ )++ self._ws = await asyncio.wait_for(+ websockets.connect(+ self.ws_uri,+ max_size=1024 * 1024 * 10, # 10MB limit+ max_queue=32,+ ping_interval=self._ping_interval,+ ping_timeout=self._ping_timeout,+ close_timeout=5,+ compression=None, # Disable compression to reduce overhead+ ),+ timeout=120,+ )+ self.logger.info("WebSocket connection established")++ # Authentication will be handled by the first command that needs it+ # Don't do authentication here to avoid recv conflicts++ self._reconnect_delay = 1 # Reset reconnect delay on successful connection+ self._last_ping = time.time()+ retry_count = 0 # Reset retry count on successful connection+ self._authenticated = False # Reset auth status on new connection++ except (asyncio.TimeoutError, websockets.exceptions.WebSocketException) as e:+ next_retry = self._reconnect_delay++ # Only log the first error at WARNING level, then every Nth attempt+ if retry_count == 1:+ self.logger.warning(+ f"Computer API Server not ready yet. Will retry automatically."+ )+ elif retry_count % log_interval == 0:+ self.logger.warning(+ f"Still waiting for Computer API Server (attempt {retry_count})..."+ )+ else:+ # All other errors are logged at DEBUG level+ self.logger.debug(f"Connection attempt {retry_count} failed: {e}")++ if self._ws:+ try:+ await self._ws.close()+ except:+ pass+ self._ws = None++ # Regular ping to check connection+ if self._ws and self._ws.state == websockets.protocol.State.OPEN:+ try:+ if time.time() - self._last_ping >= self._ping_interval:+ pong_waiter = await self._ws.ping()+ await asyncio.wait_for(pong_waiter, timeout=self._ping_timeout)+ self._last_ping = time.time()+ except Exception as e:+ self.logger.debug(f"Ping failed: {e}")+ if self._ws:+ try:+ await self._ws.close()+ except:+ pass+ self._ws = None+ continue++ await asyncio.sleep(1)++ except Exception as e:+ current_time = time.time()+ # Only log connection lost warnings at most once every min_warning_interval seconds+ if current_time - last_warning_time >= min_warning_interval:+ self.logger.warning(+ f"Computer API Server connection lost. Will retry automatically."+ )+ last_warning_time = current_time+ else:+ # Log at debug level instead+ self.logger.debug(f"Connection lost: {e}")++ if self._ws:+ try:+ await self._ws.close()+ except:+ pass+ self._ws = None++ async def _ensure_connection(self):+ """Ensure WebSocket connection is established."""+ if self._reconnect_task is None or self._reconnect_task.done():+ self._reconnect_task = asyncio.create_task(self._keep_alive())++ retry_count = 0+ max_retries = 5++ while retry_count < max_retries:+ try:+ if self._ws and self._ws.state == websockets.protocol.State.OPEN:+ return+ retry_count += 1+ await asyncio.sleep(1)+ except Exception as e:+ # Only log at ERROR level for the last retry attempt+ if retry_count == max_retries - 1:+ self.logger.error(+ f"Persistent connection check error after {retry_count} attempts: {e}"+ )+ else:+ self.logger.debug(f"Connection check error (attempt {retry_count}): {e}")+ retry_count += 1+ await asyncio.sleep(1)+ continue++ raise ConnectionError("Failed to establish WebSocket connection after multiple retries")++ async def _send_command(self, command: str, params: Optional[Dict] = None) -> Dict[str, Any]:+ """Send command through WebSocket."""+ max_retries = 3+ retry_count = 0+ last_error = None++ # Acquire lock to ensure only one command is processed at a time+ async with self._command_lock:+ self.logger.debug(f"Acquired lock for command: {command}")+ while retry_count < max_retries:+ try:+ await self._ensure_connection()+ if not self._ws:+ raise ConnectionError("WebSocket connection is not established")++ # Handle authentication if needed+ if self.api_key and self.vm_name and not self._authenticated:+ self.logger.info("Performing authentication handshake...")+ auth_message = {+ "command": "authenticate",+ "params": {+ "api_key": self.api_key,+ "container_name": self.vm_name+ }+ }+ await self._ws.send(json.dumps(auth_message))++ # Wait for authentication response+ auth_response = await asyncio.wait_for(self._ws.recv(), timeout=10)+ auth_result = json.loads(auth_response)++ if not auth_result.get("success"):+ error_msg = auth_result.get("error", "Authentication failed")+ self.logger.error(f"Authentication failed: {error_msg}")+ self._authenticated = False+ raise ConnectionError(f"Authentication failed: {error_msg}")++ self.logger.info("Authentication successful")+ self._authenticated = True++ message = {"command": command, "params": params or {}}+ await self._ws.send(json.dumps(message))+ response = await asyncio.wait_for(self._ws.recv(), timeout=30)+ self.logger.debug(f"Completed command: {command}")+ return json.loads(response)+ except Exception as e:+ last_error = e+ retry_count += 1+ if retry_count < max_retries:+ # Only log at debug level for intermediate retries+ self.logger.debug(+ f"Command '{command}' failed (attempt {retry_count}/{max_retries}): {e}"+ )+ await asyncio.sleep(1)+ continue+ else:+ # Only log at error level for the final failure+ self.logger.error(+ f"Failed to send command '{command}' after {max_retries} retries"+ )+ self.logger.debug(f"Command failure details: {e}")+ raise last_error if last_error else RuntimeError("Failed to send command")++ async def wait_for_ready(self, timeout: int = 60, interval: float = 1.0):+ """Wait for WebSocket connection to become available."""+ start_time = time.time()+ last_error = None+ attempt_count = 0+ progress_interval = 10 # Log progress every 10 seconds+ last_progress_time = start_time++ # Disable detailed logging for connection attempts+ self._log_connection_attempts = False++ try:+ self.logger.info(+ f"Waiting for Computer API Server to be ready (timeout: {timeout}s)..."+ )++ # Start the keep-alive task if it's not already running+ if self._reconnect_task is None or self._reconnect_task.done():+ self._reconnect_task = asyncio.create_task(self._keep_alive())++ # Wait for the connection to be established+ while time.time() - start_time < timeout:+ try:+ attempt_count += 1+ current_time = time.time()++ # Log progress periodically without flooding logs+ if current_time - last_progress_time >= progress_interval:+ elapsed = current_time - start_time+ self.logger.info(+ f"Still waiting for Computer API Server... (elapsed: {elapsed:.1f}s, attempts: {attempt_count})"+ )+ last_progress_time = current_time++ # Check if we have a connection+ if self._ws and self._ws.state == websockets.protocol.State.OPEN:+ # Test the connection with a simple command+ try:+ await self._send_command("get_screen_size")+ elapsed = time.time() - start_time+ self.logger.info(+ f"Computer API Server is ready (after {elapsed:.1f}s, {attempt_count} attempts)"+ )+ return # Connection is fully working+ except Exception as e:+ last_error = e+ self.logger.debug(f"Connection test failed: {e}")++ # Wait before trying again+ await asyncio.sleep(interval)++ except Exception as e:+ last_error = e+ self.logger.debug(f"Connection attempt {attempt_count} failed: {e}")+ await asyncio.sleep(interval)++ # If we get here, we've timed out+ error_msg = f"Could not connect to {self.ip_address} after {timeout} seconds"+ if last_error:+ error_msg += f": {str(last_error)}"+ self.logger.error(error_msg)+ raise TimeoutError(error_msg)+ finally:+ # Reset to default logging behavior+ self._log_connection_attempts = False++ def close(self):+ """Close WebSocket connection.++ Note: In host computer server mode, we leave the connection open+ to allow other clients to connect to the same server. The server+ will handle cleaning up idle connections.+ """+ # Only cancel the reconnect task+ if self._reconnect_task:+ self._reconnect_task.cancel()++ # Don't set closed flag or close websocket by default+ # This allows the server to stay connected for other clients+ # self._closed = True+ # if self._ws:+ # asyncio.create_task(self._ws.close())+ # self._ws = None++ def force_close(self):+ """Force close the WebSocket connection.++ This method should be called when you want to completely+ shut down the connection, not just for regular cleanup.+ """+ self._closed = True+ if self._reconnect_task:+ self._reconnect_task.cancel()+ if self._ws:+ asyncio.create_task(self._ws.close())+ self._ws = None++ # Mouse Actions+ async def mouse_down(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left") -> None:+ await self._send_command("mouse_down", {"x": x, "y": y, "button": button})++ async def mouse_up(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left") -> None:+ await self._send_command("mouse_up", {"x": x, "y": y, "button": button})++ async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:+ await self._send_command("left_click", {"x": x, "y": y})++ async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:+ await self._send_command("right_click", {"x": x, "y": y})++ async def double_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:+ await self._send_command("double_click", {"x": x, "y": y})++ async def move_cursor(self, x: int, y: int) -> None:+ await self._send_command("move_cursor", {"x": x, "y": y})++ async def drag_to(self, x: int, y: int, button: "MouseButton" = "left", duration: float = 0.5) -> None:+ await self._send_command(+ "drag_to", {"x": x, "y": y, "button": button, "duration": duration}+ )++ async def drag(self, path: List[Tuple[int, int]], button: "MouseButton" = "left", duration: float = 0.5) -> None:+ await self._send_command(+ "drag", {"path": path, "button": button, "duration": duration}+ )++ # Keyboard Actions+ async def key_down(self, key: "KeyType") -> None:+ await self._send_command("key_down", {"key": key})++ async def key_up(self, key: "KeyType") -> None:+ await self._send_command("key_up", {"key": key})++ async def type_text(self, text: str) -> None:+ # For Windows, use clipboard for Unicode text like Linux+ if any(ord(char) > 127 for char in text):+ # For Unicode text, use clipboard and paste+ await self.set_clipboard(text)+ await self.hotkey(Key.CTRL, 'v') # Windows uses Ctrl+V instead of Cmd+V+ else:+ # For ASCII text, use the regular typing method+ await self._send_command("type_text", {"text": text})++ async def press(self, key: "KeyType") -> None:+ """Press a single key.++ Args:+ key: The key to press. Can be any of:+ - A Key enum value (recommended), e.g. Key.PAGE_DOWN+ - A direct key value string, e.g. 'pagedown'+ - A single character string, e.g. 'a'++ Examples:+ ```python+ # Using enum (recommended)+ await interface.press(Key.PAGE_DOWN)+ await interface.press(Key.ENTER)++ # Using direct values+ await interface.press('pagedown')+ await interface.press('enter')++ # Using single characters+ await interface.press('a')+ ```++ Raises:+ ValueError: If the key type is invalid or the key is not recognized+ """+ if isinstance(key, Key):+ actual_key = key.value+ elif isinstance(key, str):+ # Try to convert to enum if it matches a known key+ key_or_enum = Key.from_string(key)+ actual_key = key_or_enum.value if isinstance(key_or_enum, Key) else key_or_enum+ else:+ raise ValueError(f"Invalid key type: {type(key)}. Must be Key enum or string.")++ await self._send_command("press_key", {"key": actual_key})++ async def press_key(self, key: "KeyType") -> None:+ """DEPRECATED: Use press() instead.++ This method is kept for backward compatibility but will be removed in a future version.+ Please use the press() method instead.+ """+ await self.press(key)++ async def hotkey(self, *keys: "KeyType") -> None:+ """Press multiple keys simultaneously.++ Args:+ *keys: Multiple keys to press simultaneously. Each key can be any of:+ - A Key enum value (recommended), e.g. Key.CTRL+ - A direct key value string, e.g. 'ctrl'+ - A single character string, e.g. 'a'++ Examples:+ ```python+ # Using enums (recommended)+ await interface.hotkey(Key.CTRL, Key.C) # Copy+ await interface.hotkey(Key.CTRL, Key.V) # Paste++ # Using mixed formats+ await interface.hotkey(Key.CTRL, 'a') # Select all+ ```++ Raises:+ ValueError: If any key type is invalid or not recognized+ """+ actual_keys = []+ for key in keys:+ if isinstance(key, Key):+ actual_keys.append(key.value)+ elif isinstance(key, str):+ # Try to convert to enum if it matches a known key+ key_or_enum = Key.from_string(key)+ actual_keys.append(key_or_enum.value if isinstance(key_or_enum, Key) else key_or_enum)+ else:+ raise ValueError(f"Invalid key type: {type(key)}. Must be Key enum or string.")++ await self._send_command("hotkey", {"keys": actual_keys})++ # Scrolling Actions+ async def scroll(self, x: int, y: int) -> None:+ await self._send_command("scroll", {"x": x, "y": y})++ async def scroll_down(self, clicks: int = 1) -> None:+ await self._send_command("scroll_down", {"clicks": clicks})++ async def scroll_up(self, clicks: int = 1) -> None:+ await self._send_command("scroll_up", {"clicks": clicks})++ # Screen Actions+ async def screenshot(+ self,+ boxes: Optional[List[Tuple[int, int, int, int]]] = None,+ box_color: str = "#FF0000",+ box_thickness: int = 2,+ scale_factor: float = 1.0,+ ) -> bytes:+ """Take a screenshot with optional box drawing and scaling.++ Args:+ boxes: Optional list of (x, y, width, height) tuples defining boxes to draw in screen coordinates+ box_color: Color of the boxes in hex format (default: "#FF0000" red)+ box_thickness: Thickness of the box borders in pixels (default: 2)+ scale_factor: Factor to scale the final image by (default: 1.0)+ Use > 1.0 to enlarge, < 1.0 to shrink (e.g., 0.5 for half size, 2.0 for double)++ Returns:+ bytes: The screenshot image data, optionally with boxes drawn on it and scaled+ """+ result = await self._send_command("screenshot")+ if not result.get("image_data"):+ raise RuntimeError("Failed to take screenshot")++ screenshot = decode_base64_image(result["image_data"])++ if boxes:+ # Get the natural scaling between screen and screenshot+ screen_size = await self.get_screen_size()+ screenshot_width, screenshot_height = bytes_to_image(screenshot).size+ width_scale = screenshot_width / screen_size["width"]+ height_scale = screenshot_height / screen_size["height"]++ # Scale box coordinates from screen space to screenshot space+ for box in boxes:+ scaled_box = (+ int(box[0] * width_scale), # x+ int(box[1] * height_scale), # y+ int(box[2] * width_scale), # width+ int(box[3] * height_scale), # height+ )+ screenshot = draw_box(+ screenshot,+ x=scaled_box[0],+ y=scaled_box[1],+ width=scaled_box[2],+ height=scaled_box[3],+ color=box_color,+ thickness=box_thickness,+ )++ if scale_factor != 1.0:+ screenshot = resize_image(screenshot, scale_factor)++ return screenshot++ async def get_screen_size(self) -> Dict[str, int]:+ result = await self._send_command("get_screen_size")+ if result["success"] and result["size"]:+ return result["size"]+ raise RuntimeError("Failed to get screen size")++ async def get_cursor_position(self) -> Dict[str, int]:+ result = await self._send_command("get_cursor_position")+ if result["success"] and result["position"]:+ return result["position"]+ raise RuntimeError("Failed to get cursor position")++ # Clipboard Actions+ async def copy_to_clipboard(self) -> str:+ result = await self._send_command("copy_to_clipboard")+ if result["success"] and result["content"]:+ return result["content"]+ raise RuntimeError("Failed to get clipboard content")++ async def set_clipboard(self, text: str) -> None:+ await self._send_command("set_clipboard", {"text": text})++ # File System Actions+ async def file_exists(self, path: str) -> bool:+ result = await self._send_command("file_exists", {"path": path})+ return result.get("exists", False)++ async def directory_exists(self, path: str) -> bool:+ result = await self._send_command("directory_exists", {"path": path})+ return result.get("exists", False)++ async def list_dir(self, path: str) -> list[str]:+ result = await self._send_command("list_dir", {"path": path})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to list directory"))+ return result.get("files", [])++ async def read_text(self, path: str) -> str:+ result = await self._send_command("read_text", {"path": path})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to read file"))+ return result.get("content", "")++ async def write_text(self, path: str, content: str) -> None:+ result = await self._send_command("write_text", {"path": path, "content": content})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to write file"))++ async def read_bytes(self, path: str) -> bytes:+ result = await self._send_command("read_bytes", {"path": path})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to read file"))+ content_b64 = result.get("content_b64", "")+ return decode_base64_image(content_b64)++ async def write_bytes(self, path: str, content: bytes) -> None:+ result = await self._send_command("write_bytes", {"path": path, "content_b64": encode_base64_image(content)})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to write file"))++ async def delete_file(self, path: str) -> None:+ result = await self._send_command("delete_file", {"path": path})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to delete file"))++ async def create_dir(self, path: str) -> None:+ result = await self._send_command("create_dir", {"path": path})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to create directory"))++ async def delete_dir(self, path: str) -> None:+ result = await self._send_command("delete_dir", {"path": path})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to delete directory"))++ async def run_command(self, command: str) -> Tuple[str, str]:+ result = await self._send_command("run_command", {"command": command})+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to run command"))+ return result.get("stdout", ""), result.get("stderr", "")++ # Accessibility Actions+ async def get_accessibility_tree(self) -> Dict[str, Any]:+ """Get the accessibility tree of the current screen."""+ result = await self._send_command("get_accessibility_tree")+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to get accessibility tree"))+ return result++ async def get_active_window_bounds(self) -> Dict[str, int]:+ """Get the bounds of the currently active window."""+ result = await self._send_command("get_active_window_bounds")+ if result["success"] and result["bounds"]:+ return result["bounds"]+ raise RuntimeError("Failed to get active window bounds")++ async def to_screen_coordinates(self, x: float, y: float) -> tuple[float, float]:+ """Convert screenshot coordinates to screen coordinates.++ Args:+ x: X coordinate in screenshot space+ y: Y coordinate in screenshot space++ Returns:+ tuple[float, float]: (x, y) coordinates in screen space+ """+ screen_size = await self.get_screen_size()+ screenshot = await self.screenshot()+ screenshot_img = bytes_to_image(screenshot)+ screenshot_width, screenshot_height = screenshot_img.size++ # Calculate scaling factors+ width_scale = screen_size["width"] / screenshot_width+ height_scale = screen_size["height"] / screenshot_height++ # Convert coordinates+ screen_x = x * width_scale+ screen_y = y * height_scale++ return screen_x, screen_y++ async def to_screenshot_coordinates(self, x: float, y: float) -> tuple[float, float]:+ """Convert screen coordinates to screenshot coordinates.++ Args:+ x: X coordinate in screen space+ y: Y coordinate in screen space++ Returns:+ tuple[float, float]: (x, y) coordinates in screenshot space+ """+ screen_size = await self.get_screen_size()+ screenshot = await self.screenshot()+ screenshot_img = bytes_to_image(screenshot)+ screenshot_width, screenshot_height = screenshot_img.size++ # Calculate scaling factors+ width_scale = screenshot_width / screen_size["width"]+ height_scale = screenshot_height / screen_size["height"]++ # Convert coordinates+ screenshot_x = x * width_scale+ screenshot_y = y * height_scale++ return screenshot_x, screenshot_y