Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(agent): add defensive coordinate scaling for high-res VMs (#1108)
Sarina Li3cae8267c92b
diff --git a/libs/python/agent/agent/loops/anthropic.py b/libs/python/agent/agent/loops/anthropic.pyindex 9f532c04e1..ec90895483 100644--- a/libs/python/agent/agent/loops/anthropic.py+++ b/libs/python/agent/agent/loops/anthropic.py@@ -4,6 +4,7 @@ Anthropic hosted tools agent loop implementation using liteLLMimport asyncioimport json+import loggingfrom typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Unionimport litellm@@ -32,6 +33,23 @@ from ..responses import ()from ..types import AgentCapability, AgentResponse, Messages, Tools+logger = logging.getLogger(__name__)++# Recommended maximum resolution for Anthropic's computer-use API.+# Screenshots larger than this are internally downscaled by the API, causing+# coordinate mismatches. We proactively downscale to avoid the offset.+RECOMMENDED_MAX_WIDTH = 1024+RECOMMENDED_MAX_HEIGHT = 768+++def _scale_coordinate(coord: int, scale: float) -> int:+ """Scale a single coordinate value by a factor and round to int."""+ scaled = int(round(coord * scale))+ if scale != 1.0:+ logger.debug("Scaling coordinate: %d * %.4f -> %d", coord, scale, scaled)+ return scaled++# Model version mapping to tool version and beta flagMODEL_TOOL_MAPPING = [# Claude Opus 4.6/4.5 and Sonnet 4.6 require the 2025-11-24 computer-use beta@@ -82,6 +100,24 @@ async def _map_computer_tool_to_anthropic(computer_tool: Any, tool_version: str)# Fallback to default dimensions if method failswidth, height = 1024, 768+ # Cap dimensions to recommended max so they match downscaled screenshots+ if width > RECOMMENDED_MAX_WIDTH or height > RECOMMENDED_MAX_HEIGHT:+ scale = min(RECOMMENDED_MAX_WIDTH / width, RECOMMENDED_MAX_HEIGHT / height)+ new_width = int(width * scale)+ new_height = int(height * scale)+ logger.debug(+ "Capping tool dimensions: %dx%d -> %dx%d (scale=%.4f)",+ width,+ height,+ new_width,+ new_height,+ scale,+ )+ width = new_width+ height = new_height+ else:+ logger.debug("Tool dimensions within limits: %dx%d", width, height)+return {"type": tool_version,"function": {@@ -122,10 +158,19 @@ async def _prepare_tools_for_anthropic(tool_schemas: List[Dict[str, Any]], modelreturn anthropic_tools-def _convert_responses_items_to_completion_messages(messages: Messages) -> List[Dict[str, Any]]:- """Convert responses_items message format to liteLLM completion format."""+def _convert_responses_items_to_completion_messages(+ messages: Messages,+) -> Tuple[List[Dict[str, Any]], Tuple[float, float]]:+ """Convert responses_items message format to liteLLM completion format.++ Returns:+ A tuple of (completion_messages, scale_factors) where scale_factors is+ (scale_x, scale_y) representing the ratio of original to downscaled+ dimensions. Use these to upscale coordinates returned by the API.+ """completion_messages = []call_id_to_fn_name = {}+ scale_factors: Tuple[float, float] = (1.0, 1.0)for message in messages:msg_type = message.get("type")@@ -627,6 +672,55 @@ def _convert_responses_items_to_completion_messages(messages: Messages) -> List[if output.get("type") == "input_image":# Screenshot result - convert to OpenAI format with image_url contentimage_url = output.get("image_url", "")++ # Reset scale factors for each new screenshot so stale values+ # from a previous downscale don't carry over.+ scale_factors = (1.0, 1.0)++ # Downscale screenshot if it exceeds recommended max resolution+ if image_url and image_url.startswith("data:"):+ try:+ import base64+ from io import BytesIO++ from PIL import Image++ # Extract base64 data after the header+ header, b64_data = image_url.split(",", 1)+ img_bytes = base64.b64decode(b64_data)+ img = Image.open(BytesIO(img_bytes))+ orig_w, orig_h = img.size++ if orig_w > RECOMMENDED_MAX_WIDTH or orig_h > RECOMMENDED_MAX_HEIGHT:+ scale = min(+ RECOMMENDED_MAX_WIDTH / orig_w,+ RECOMMENDED_MAX_HEIGHT / orig_h,+ )+ new_w = int(orig_w * scale)+ new_h = int(orig_h * scale)+ img = img.resize((new_w, new_h), Image.LANCZOS)++ # Re-encode to base64+ buf = BytesIO()+ img.save(buf, format="PNG")+ new_b64 = base64.b64encode(buf.getvalue()).decode("utf-8")+ image_url = f"{header},{new_b64}"++ # Store scale factors for coordinate upscaling+ scale_factors = (orig_w / new_w, orig_h / new_h)+ logger.debug(+ "Downscaled screenshot: %dx%d -> %dx%d (scale=%.4f, upscale_factors=%.4fx%.4f)",+ orig_w,+ orig_h,+ new_w,+ new_h,+ scale,+ scale_factors[0],+ scale_factors[1],+ )+ except Exception:+ pass # If downscaling fails, send original image+completion_messages.append({"role": "function",@@ -646,11 +740,21 @@ def _convert_responses_items_to_completion_messages(messages: Messages) -> List[})- return completion_messages+ return completion_messages, scale_factors-def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]]:- """Convert liteLLM completion response to responses_items message format."""+def _convert_completion_to_responses_items(+ response: Any,+ scale_x: float = 1.0,+ scale_y: float = 1.0,+) -> List[Dict[str, Any]]:+ """Convert liteLLM completion response to responses_items message format.++ Args:+ response: The liteLLM completion response.+ scale_x: Horizontal upscale factor for coordinates (original_w / downscaled_w).+ scale_y: Vertical upscale factor for coordinates (original_h / downscaled_h).+ """responses_items = []if not response or not hasattr(response, "choices") or not response.choices:@@ -699,8 +803,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = tool_input.get("coordinate", [0, 0])responses_items.append(make_click_item(- x=coordinate[0] if len(coordinate) > 0 else 0,- y=coordinate[1] if len(coordinate) > 1 else 0,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if len(coordinate) > 0+ else 0+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if len(coordinate) > 1+ else 0+ ),call_id=call_id,))@@ -722,8 +834,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = tool_input.get("coordinate", [0, 0])responses_items.append(make_move_item(- x=coordinate[0] if len(coordinate) > 0 else 0,- y=coordinate[1] if len(coordinate) > 1 else 0,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if len(coordinate) > 0+ else 0+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if len(coordinate) > 1+ else 0+ ),call_id=call_id,))@@ -752,8 +872,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any])responses_items.append(make_scroll_item(- x=coordinate[0] if len(coordinate) > 0 else 0,- y=coordinate[1] if len(coordinate) > 1 else 0,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if len(coordinate) > 0+ else 0+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if len(coordinate) > 1+ else 0+ ),scroll_x=scroll_x,scroll_y=scroll_y,call_id=call_id,@@ -766,12 +894,28 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]make_drag_item(path=[{- "x": start_coord[0] if len(start_coord) > 0 else 0,- "y": start_coord[1] if len(start_coord) > 1 else 0,+ "x": (+ _scale_coordinate(start_coord[0], scale_x)+ if len(start_coord) > 0+ else 0+ ),+ "y": (+ _scale_coordinate(start_coord[1], scale_y)+ if len(start_coord) > 1+ else 0+ ),},{- "x": end_coord[0] if len(end_coord) > 0 else 0,- "y": end_coord[1] if len(end_coord) > 1 else 0,+ "x": (+ _scale_coordinate(end_coord[0], scale_x)+ if len(end_coord) > 0+ else 0+ ),+ "y": (+ _scale_coordinate(end_coord[1], scale_y)+ if len(end_coord) > 1+ else 0+ ),},],call_id=call_id,@@ -781,8 +925,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = tool_input.get("coordinate", [0, 0])responses_items.append(make_click_item(- x=coordinate[0] if len(coordinate) > 0 else 0,- y=coordinate[1] if len(coordinate) > 1 else 0,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if len(coordinate) > 0+ else 0+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if len(coordinate) > 1+ else 0+ ),button="right",call_id=call_id,)@@ -791,8 +943,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = tool_input.get("coordinate", [0, 0])responses_items.append(make_click_item(- x=coordinate[0] if len(coordinate) > 0 else 0,- y=coordinate[1] if len(coordinate) > 1 else 0,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if len(coordinate) > 0+ else 0+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if len(coordinate) > 1+ else 0+ ),button="wheel",call_id=call_id,)@@ -801,8 +961,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = tool_input.get("coordinate", [0, 0])responses_items.append(make_double_click_item(- x=coordinate[0] if len(coordinate) > 0 else 0,- y=coordinate[1] if len(coordinate) > 1 else 0,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if len(coordinate) > 0+ else 0+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if len(coordinate) > 1+ else 0+ ),call_id=call_id,))@@ -833,8 +1001,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = tool_input.get("coordinate", [None, None])responses_items.append(make_left_mouse_down_item(- x=coordinate[0] if len(coordinate) > 0 else None,- y=coordinate[1] if len(coordinate) > 1 else None,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if (len(coordinate) > 0 and coordinate[0] is not None)+ else None+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if (len(coordinate) > 1 and coordinate[1] is not None)+ else None+ ),call_id=call_id,))@@ -853,8 +1029,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = tool_input.get("coordinate", [None, None])responses_items.append(make_left_mouse_up_item(- x=coordinate[0] if len(coordinate) > 0 else None,- y=coordinate[1] if len(coordinate) > 1 else None,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if (len(coordinate) > 0 and coordinate[0] is not None)+ else None+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if (len(coordinate) > 1 and coordinate[1] is not None)+ else None+ ),call_id=call_id,))@@ -961,8 +1145,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = args.get("coordinate", [0, 0])responses_items.append(make_click_item(- x=coordinate[0] if len(coordinate) > 0 else 0,- y=coordinate[1] if len(coordinate) > 1 else 0,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if len(coordinate) > 0+ else 0+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if len(coordinate) > 1+ else 0+ ),call_id=call_id,))@@ -1048,8 +1240,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = args.get("coordinate", [0, 0])responses_items.append(make_move_item(- x=coordinate[0] if len(coordinate) > 0 else 0,- y=coordinate[1] if len(coordinate) > 1 else 0,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if len(coordinate) > 0+ else 0+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if len(coordinate) > 1+ else 0+ ),call_id=call_id,))@@ -1098,8 +1298,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any])responses_items.append(make_scroll_item(- x=coordinate[0] if len(coordinate) > 0 else 0,- y=coordinate[1] if len(coordinate) > 1 else 0,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if len(coordinate) > 0+ else 0+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if len(coordinate) > 1+ else 0+ ),scroll_x=scroll_x,scroll_y=scroll_y,call_id=call_id,@@ -1138,12 +1346,28 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]make_drag_item(path=[{- "x": start_coord[0] if len(start_coord) > 0 else 0,- "y": start_coord[1] if len(start_coord) > 1 else 0,+ "x": (+ _scale_coordinate(start_coord[0], scale_x)+ if len(start_coord) > 0+ else 0+ ),+ "y": (+ _scale_coordinate(start_coord[1], scale_y)+ if len(start_coord) > 1+ else 0+ ),},{- "x": end_coord[0] if len(end_coord) > 0 else 0,- "y": end_coord[1] if len(end_coord) > 1 else 0,+ "x": (+ _scale_coordinate(end_coord[0], scale_x)+ if len(end_coord) > 0+ else 0+ ),+ "y": (+ _scale_coordinate(end_coord[1], scale_y)+ if len(end_coord) > 1+ else 0+ ),},],call_id=call_id,@@ -1177,8 +1401,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = args.get("coordinate", [0, 0])responses_items.append(make_click_item(- x=coordinate[0] if len(coordinate) > 0 else 0,- y=coordinate[1] if len(coordinate) > 1 else 0,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if len(coordinate) > 0+ else 0+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if len(coordinate) > 1+ else 0+ ),button="right",call_id=call_id,)@@ -1211,8 +1443,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = args.get("coordinate", [0, 0])responses_items.append(make_click_item(- x=coordinate[0] if len(coordinate) > 0 else 0,- y=coordinate[1] if len(coordinate) > 1 else 0,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if len(coordinate) > 0+ else 0+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if len(coordinate) > 1+ else 0+ ),button="wheel",call_id=call_id,)@@ -1244,8 +1484,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = args.get("coordinate", [0, 0])responses_items.append(make_double_click_item(- x=coordinate[0] if len(coordinate) > 0 else 0,- y=coordinate[1] if len(coordinate) > 1 else 0,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if len(coordinate) > 0+ else 0+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if len(coordinate) > 1+ else 0+ ),call_id=call_id,))@@ -1302,8 +1550,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = args.get("coordinate", [None, None])responses_items.append(make_left_mouse_down_item(- x=coordinate[0] if len(coordinate) > 0 else None,- y=coordinate[1] if len(coordinate) > 1 else None,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if (len(coordinate) > 0 and coordinate[0] is not None)+ else None+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if (len(coordinate) > 1 and coordinate[1] is not None)+ else None+ ),call_id=call_id,))@@ -1335,8 +1591,16 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]coordinate = args.get("coordinate", [None, None])responses_items.append(make_left_mouse_up_item(- x=coordinate[0] if len(coordinate) > 0 else None,- y=coordinate[1] if len(coordinate) > 1 else None,+ x=(+ _scale_coordinate(coordinate[0], scale_x)+ if (len(coordinate) > 0 and coordinate[0] is not None)+ else None+ ),+ y=(+ _scale_coordinate(coordinate[1], scale_y)+ if (len(coordinate) > 1 and coordinate[1] is not None)+ else None+ ),call_id=call_id,))@@ -1519,7 +1783,10 @@ class AnthropicHostedToolsConfig(AsyncAgentConfig):anthropic_tools = await _prepare_tools_for_anthropic(tools, model)# Convert responses_items messages to completion format- completion_messages = _convert_responses_items_to_completion_messages(messages)+ completion_messages, scale_factors = _convert_responses_items_to_completion_messages(+ messages+ )+ scale_x, scale_y = scale_factorsif use_prompt_caching:# First combine messages to reduce number of blockscompletion_messages = _combine_completion_messages(completion_messages)@@ -1551,8 +1818,10 @@ class AnthropicHostedToolsConfig(AsyncAgentConfig):if _on_api_end:await _on_api_end(api_kwargs, response)- # Convert response to responses_items format- responses_items = _convert_completion_to_responses_items(response)+ # Convert response to responses_items format, upscaling coordinates if needed+ responses_items = _convert_completion_to_responses_items(+ response, scale_x=scale_x, scale_y=scale_y+ )# Extract usage informationresponses_usage = {diff --git a/libs/python/computer/computer/interface/generic.py b/libs/python/computer/computer/interface/generic.pyindex ebecda0b19..fc58057604 100644--- a/libs/python/computer/computer/interface/generic.py+++ b/libs/python/computer/computer/interface/generic.py@@ -308,6 +308,16 @@ class GenericComputerInterface(BaseComputerInterface):width_scale = screenshot_width / screen_size["width"]height_scale = screenshot_height / screen_size["height"]+ self.logger.debug(+ "Screenshot box scaling: screen=%dx%d, screenshot=%dx%d, scale=%.4fx%.4f",+ screen_size["width"],+ screen_size["height"],+ screenshot_width,+ screenshot_height,+ width_scale,+ height_scale,+ )+# Scale box coordinates from screen space to screenshot spacefor box in boxes:scaled_box = (@@ -316,6 +326,17 @@ class GenericComputerInterface(BaseComputerInterface):int(box[2] * width_scale), # widthint(box[3] * height_scale), # height)+ self.logger.debug(+ "Scaling box: (%d,%d,%d,%d) -> (%d,%d,%d,%d)",+ box[0],+ box[1],+ box[2],+ box[3],+ scaled_box[0],+ scaled_box[1],+ scaled_box[2],+ scaled_box[3],+ )screenshot = draw_box(screenshot,x=scaled_box[0],@@ -327,6 +348,7 @@ class GenericComputerInterface(BaseComputerInterface):)if scale_factor != 1.0:+ self.logger.debug("Resizing screenshot with scale_factor=%.4f", scale_factor)screenshot = resize_image(screenshot, scale_factor)return screenshot@@ -649,6 +671,21 @@ class GenericComputerInterface(BaseComputerInterface):screen_x = x * width_scalescreen_y = y * height_scale+ self.logger.debug(+ "to_screen_coordinates: (%.1f,%.1f) -> (%.1f,%.1f) "+ "(screen=%dx%d, screenshot=%dx%d, scale=%.4fx%.4f)",+ x,+ y,+ screen_x,+ screen_y,+ screen_size["width"],+ screen_size["height"],+ screenshot_width,+ screenshot_height,+ width_scale,+ height_scale,+ )+return screen_x, screen_yasync def to_screenshot_coordinates(self, x: float, y: float) -> tuple[float, float]:@@ -674,6 +711,21 @@ class GenericComputerInterface(BaseComputerInterface):screenshot_x = x * width_scalescreenshot_y = y * height_scale+ self.logger.debug(+ "to_screenshot_coordinates: (%.1f,%.1f) -> (%.1f,%.1f) "+ "(screen=%dx%d, screenshot=%dx%d, scale=%.4fx%.4f)",+ x,+ y,+ screenshot_x,+ screenshot_y,+ screen_size["width"],+ screen_size["height"],+ screenshot_width,+ screenshot_height,+ width_scale,+ height_scale,+ )+return screenshot_x, screenshot_y# Playwright browser control