Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
agent fixes for scaling resolution on gemini + prefxing issues (#1117)
Sarina Lia5495b294c02
diff --git a/docs/content/docs/cua/reference/agent-sdk/index.mdx b/docs/content/docs/cua/reference/agent-sdk/index.mdxindex 27a291ef95..7ab38a8ff6 100644--- a/docs/content/docs/cua/reference/agent-sdk/index.mdx+++ b/docs/content/docs/cua/reference/agent-sdk/index.mdx@@ -7,7 +7,7 @@ description: Python API reference for building computer-use agentsAUTO-GENERATED FILE - DO NOT EDIT DIRECTLYGenerated by: npx tsx scripts/docs-generators/python-sdk.tsSource: libs/python/agent/agent- Version: 0.7.27+ Version: 0.7.29*/}import { Callout } from 'fumadocs-ui/components/callout';@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';<VersionHeaderversions={[{"version":"0.7","href":"/cua/reference/agent-sdk","isCurrent":true},{"version":"0.6","href":"/cua/reference/agent-sdk/v0.6/api","isCurrent":false},{"version":"0.5","href":"/cua/reference/agent-sdk/v0.5/api","isCurrent":false},{"version":"0.4","href":"/cua/reference/agent-sdk/v0.4/api","isCurrent":false},{"version":"0.3","href":"/cua/reference/agent-sdk/v0.3/api","isCurrent":false}]}currentVersion="0.7"- fullVersion="0.7.27"+ fullVersion="0.7.29"packageName="cua-agent"/>diff --git a/docs/content/docs/cua/reference/cli/index.mdx b/docs/content/docs/cua/reference/cli/index.mdxindex 352721ac38..e24add0bfe 100644--- a/docs/content/docs/cua/reference/cli/index.mdx+++ b/docs/content/docs/cua/reference/cli/index.mdx@@ -7,7 +7,7 @@ description: Python API reference for the Cua command-line interfaceAUTO-GENERATED FILE - DO NOT EDIT DIRECTLYGenerated by: npx tsx scripts/docs-generators/python-sdk.tsSource: libs/python/cua-cli/cua_cli- Version: 0.1.5+ Version: 0.1.6*/}import { Callout } from 'fumadocs-ui/components/callout';@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';<VersionHeaderversions={[{"version":"0.1","href":"/cua/reference/cli","isCurrent":true}]}currentVersion="0.1"- fullVersion="0.1.5"+ fullVersion="0.1.6"packageName="cua-cli"/>diff --git a/docs/content/docs/cua/reference/computer-sdk/index.mdx b/docs/content/docs/cua/reference/computer-sdk/index.mdxindex a8b5c57143..89d2d7e013 100644--- a/docs/content/docs/cua/reference/computer-sdk/index.mdx+++ b/docs/content/docs/cua/reference/computer-sdk/index.mdx@@ -7,7 +7,7 @@ description: Python API reference for controlling virtual machines and computerAUTO-GENERATED FILE - DO NOT EDIT DIRECTLYGenerated by: npx tsx scripts/docs-generators/python-sdk.tsSource: libs/python/computer/computer- Version: 0.5.14+ Version: 0.5.16*/}import { Callout } from 'fumadocs-ui/components/callout';@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';<VersionHeaderversions={[{"version":"0.5","href":"/cua/reference/computer-sdk","isCurrent":true},{"version":"0.4","href":"/cua/reference/computer-sdk/v0.4/api","isCurrent":false},{"version":"0.3","href":"/cua/reference/computer-sdk/v0.3/api","isCurrent":false}]}currentVersion="0.5"- fullVersion="0.5.14"+ fullVersion="0.5.16"packageName="cua-computer"/>@@ -559,707 +559,707 @@ the returned :class:`PtyHandle` reflects any resizes since creation.----## diorama_computer+## tracing++Computer tracing functionality for recording sessions.++This module provides a Computer.tracing API inspired by Playwright's tracing functionality,+allowing users to record computer interactions for debugging, training, and analysis.----## Key+## ComputerTracing-*Inherits from: Enum*+Computer tracing class that records computer interactions and saves them to disk.-Keyboard keys that can be used with press_key.+This class provides a flexible API for recording computer sessions with configurable+options for what to record (screenshots, API calls, video, etc.).-These key names follow a consistent cross-platform keyboard key naming convention.+### Constructor++```python+ComputerTracing(self, computer_instance)+```### Attributes| Name | Type | Description ||------|------|-------------|-| `PAGE_DOWN` | `Any` | |-| `PAGE_UP` | `Any` | |-| `HOME` | `Any` | |-| `END` | `Any` | |-| `LEFT` | `Any` | |-| `RIGHT` | `Any` | |-| `UP` | `Any` | |-| `DOWN` | `Any` | |-| `RETURN` | `Any` | |-| `ENTER` | `Any` | |-| `ESCAPE` | `Any` | |-| `ESC` | `Any` | |-| `TAB` | `Any` | |-| `SPACE` | `Any` | |-| `BACKSPACE` | `Any` | |-| `DELETE` | `Any` | |-| `ALT` | `Any` | |-| `CTRL` | `Any` | |-| `SHIFT` | `Any` | |-| `WIN` | `Any` | |-| `COMMAND` | `Any` | |-| `OPTION` | `Any` | |-| `F1` | `Any` | |-| `F2` | `Any` | |-| `F3` | `Any` | |-| `F4` | `Any` | |-| `F5` | `Any` | |-| `F6` | `Any` | |-| `F7` | `Any` | |-| `F8` | `Any` | |-| `F9` | `Any` | |-| `F10` | `Any` | |-| `F11` | `Any` | |-| `F12` | `Any` | |+| `is_tracing` | `bool` | Check if tracing is currently active. |### Methods-#### Key.from_string+#### ComputerTracing.start```python-def from_string(cls, key: str) -> Key | str+async def start(self, config: Optional[Dict[str, Any]] = None) -> None```-Convert a string key name to a Key enum value.+Start tracing with the specified configuration.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `key` | `Any` | String key name to convert |+| `config` | `Any` | Tracing configuration dict with options: - video: bool - Record video frames (default: False) - screenshots: bool - Record screenshots (default: True) - api_calls: bool - Record API calls and results (default: True) - accessibility_tree: bool - Record accessibility tree snapshots (default: False) - metadata: bool - Record custom metadata (default: True) - name: str - Custom trace name (default: auto-generated) - path: str - Custom trace directory path (default: auto-generated) |-**Returns:** Key enum value if the string matches a known key, otherwise returns the original string for single character keys+#### ComputerTracing.stop----+```python+async def stop(self, options: Optional[Dict[str, Any]] = None) -> str+```-## DioramaComputer+Stop tracing and save the trace data.-A Computer-compatible proxy for Diorama that sends commands over the ComputerInterface.+**Parameters:**-### Constructor+| Name | Type | Description |+|------|------|-------------|+| `options` | `Any` | Stop options dict with: - path: str - Custom output path for the trace archive - format: str - Output format ('zip' or 'dir', default: 'zip') |++**Returns:** str: Path to the saved trace file or directory++#### ComputerTracing.record_api_call```python-DioramaComputer(self, computer, apps)+async def record_api_call(self, method: str, args: Dict[str, Any], result: Any = None, error: Optional[Exception] = None) -> None```-### Attributes+Record an API call event.++**Parameters:**| Name | Type | Description ||------|------|-------------|-| `computer` | `Any` | |-| `apps` | `Any` | |-| `interface` | `Any` | |+| `method` | `Any` | The method name that was called |+| `args` | `Any` | Arguments passed to the method |+| `result` | `Any` | Result returned by the method |+| `error` | `Any` | Exception raised by the method, if any |-### Methods+#### ComputerTracing.record_accessibility_tree-#### DioramaComputer.run+```python+async def record_accessibility_tree(self) -> None+```++Record the current accessibility tree if enabled.++#### ComputerTracing.add_metadata```python-async def run(self)+async def add_metadata(self, key: str, value: Any) -> None```-Initialize and run the DioramaComputer if not already initialized.+Add custom metadata to the trace.-**Returns:** self: The DioramaComputer instance+**Parameters:**++| Name | Type | Description |+|------|------|-------------|+| `key` | `Any` | Metadata key |+| `value` | `Any` | Metadata value |----## DioramaComputerInterface+## models-Diorama Interface proxy that sends diorama_cmds via the Computer's interface.+Models for computer configuration.-### Constructor+----```python-DioramaComputerInterface(self, computer, apps)-```+## BaseVMProvider++*Inherits from: AsyncContextManager*++Base interface for VM providers.++All VM provider implementations must implement this interface.### Attributes| Name | Type | Description ||------|------|-------------|-| `computer` | `Any` | |-| `apps` | `Any` | |+| `provider_type` | `VMProviderType` | Get the provider type. |### Methods-#### DioramaComputerInterface.screenshot+#### BaseVMProvider.get_vm```python-async def screenshot(self, as_bytes = True)+async def get_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]```-Take a screenshot of the diorama scene.+Get VM information by name.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `as_bytes` | `bool` | If True, return image as bytes; if False, return PIL Image object |+| `name` | `Any` | Name of the VM to get information for |+| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |-**Returns:** bytes or PIL.Image: Screenshot data in the requested format+**Returns:** Dictionary with VM information including status, IP address, etc.-#### DioramaComputerInterface.get_screen_size+#### BaseVMProvider.list_vms```python-async def get_screen_size(self)+async def list_vms(self) -> ListVMsResponse```-Get the dimensions of the diorama scene.+List all available VMs.-**Returns:** dict: Dictionary containing 'width' and 'height' keys with pixel dimensions+**Returns:** ListVMsResponse: A list of minimal VM objects as defined in `computer.providers.types.MinimalVM`.-#### DioramaComputerInterface.move_cursor+#### BaseVMProvider.run_vm```python-async def move_cursor(self, x, y)+async def run_vm(self, image: str, name: str, run_opts: Dict[str, Any], storage: Optional[str] = None) -> Dict[str, Any]```-Move the cursor to the specified coordinates.+Run a VM by name with the given options.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `x` | `int` | X coordinate to move cursor to |-| `y` | `int` | Y coordinate to move cursor to |+| `image` | `Any` | Name/tag of the image to use |+| `name` | `Any` | Name of the VM to run |+| `run_opts` | `Any` | Dictionary of run options (memory, cpu, etc.) |+| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |-#### DioramaComputerInterface.left_click+**Returns:** Dictionary with VM run status and information++#### BaseVMProvider.stop_vm```python-async def left_click(self, x = None, y = None)+async def stop_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]```-Perform a left mouse click at the specified coordinates or current cursor position.+Stop a VM by name.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `x` | `int, optional` | X coordinate to click at. If None, clicks at current cursor position |-| `y` | `int, optional` | Y coordinate to click at. If None, clicks at current cursor position |+| `name` | `Any` | Name of the VM to stop |+| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |-#### DioramaComputerInterface.right_click+**Returns:** Dictionary with VM stop status and information++#### BaseVMProvider.restart_vm```python-async def right_click(self, x = None, y = None)+async def restart_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]```-Perform a right mouse click at the specified coordinates or current cursor position.+Restart a VM by name.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `x` | `int, optional` | X coordinate to click at. If None, clicks at current cursor position |-| `y` | `int, optional` | Y coordinate to click at. If None, clicks at current cursor position |+| `name` | `Any` | Name of the VM to restart |+| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |-#### DioramaComputerInterface.double_click+**Returns:** Dictionary with VM restart status and information++#### BaseVMProvider.update_vm```python-async def double_click(self, x = None, y = None)+async def update_vm(self, name: str, update_opts: Dict[str, Any], storage: Optional[str] = None) -> Dict[str, Any]```-Perform a double mouse click at the specified coordinates or current cursor position.+Update VM configuration.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `x` | `int, optional` | X coordinate to double-click at. If None, clicks at current cursor position |-| `y` | `int, optional` | Y coordinate to double-click at. If None, clicks at current cursor position |+| `name` | `Any` | Name of the VM to update |+| `update_opts` | `Any` | Dictionary of update options (memory, cpu, etc.) |+| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |-#### DioramaComputerInterface.scroll_up+**Returns:** Dictionary with VM update status and information++#### BaseVMProvider.get_ip```python-async def scroll_up(self, clicks = 1)+async def get_ip(self, name: str, storage: Optional[str] = None, retry_delay: int = 2) -> str```-Scroll up by the specified number of clicks.+Get the IP address of a VM, waiting indefinitely until it's available.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `clicks` | `int` | Number of scroll clicks to perform upward. Defaults to 1 |--#### DioramaComputerInterface.scroll_down+| `name` | `Any` | Name of the VM to get the IP for |+| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |+| `retry_delay` | `Any` | Delay between retries in seconds (default: 2) |-```python-async def scroll_down(self, clicks = 1)-```+**Returns:** IP address of the VM when it becomes available-Scroll down by the specified number of clicks.+----**Parameters:**+## Display-| Name | Type | Description |-|------|------|-------------|-| `clicks` | `int` | Number of scroll clicks to perform downward. Defaults to 1 |+Display configuration.-#### DioramaComputerInterface.drag_to+### Constructor```python-async def drag_to(self, x, y, duration = 0.5)+Display(self, width: int, height: int) -> None```-Drag from the current cursor position to the specified coordinates.--**Parameters:**+### Attributes| Name | Type | Description ||------|------|-------------|-| `x` | `int` | X coordinate to drag to |-| `y` | `int` | Y coordinate to drag to |-| `duration` | `float` | Duration of the drag operation in seconds. Defaults to 0.5 |--#### DioramaComputerInterface.get_cursor_position+| `width` | `int` | |+| `height` | `int` | |-```python-async def get_cursor_position(self)-```+----Get the current cursor position.+## Image-**Returns:** dict: Dictionary containing the current cursor coordinates+VM image configuration.-#### DioramaComputerInterface.type_text+### Constructor```python-async def type_text(self, text)+Image(self, image: str, tag: str, name: str) -> None```-Type the specified text at the current cursor position.--**Parameters:**+### Attributes| Name | Type | Description ||------|------|-------------|-| `text` | `str` | The text to type |--#### DioramaComputerInterface.press_key--```python-async def press_key(self, key)-```+| `image` | `str` | |+| `tag` | `str` | |+| `name` | `str` | |-Press a single key.+----**Parameters:**+## Computer-| Name | Type | Description |-|------|------|-------------|-| `key` | `Any` | The key to press |+Computer configuration.-#### DioramaComputerInterface.hotkey+### Constructor```python-async def hotkey(self, keys = ())+Computer(self, image: str, tag: str, name: str, display: Display, memory: str, cpu: str, vm_provider: Optional[BaseVMProvider] = None) -> None```-Press multiple keys simultaneously as a hotkey combination.+### Attributes-**Raises:**+| Name | Type | Description |+|------|------|-------------|+| `image` | `str` | |+| `tag` | `str` | |+| `name` | `str` | |+| `display` | `Display` | |+| `memory` | `str` | |+| `cpu` | `str` | |+| `vm_provider` | `Optional[BaseVMProvider]` | |-- `ValueError` - If any key is not a Key enum or string type+### Methods-#### DioramaComputerInterface.to_screen_coordinates+#### Computer.get_ip```python-async def to_screen_coordinates(self, x, y)+async def get_ip(self) -> Optional[str]```-Convert coordinates to screen coordinates.--**Parameters:**+Get the IP address of the VM.-| Name | Type | Description |-|------|------|-------------|-| `x` | `int` | X coordinate to convert |-| `y` | `int` | Y coordinate to convert |+----**Returns:** dict: Dictionary containing the converted screen coordinates+## diorama_computer----## helpers--Helper functions and decorators for the Computer module.+## Key----+*Inherits from: Enum*-## DependencyInfo+Keyboard keys that can be used with press_key.-*Inherits from: TypedDict*+These key names follow a consistent cross-platform keyboard key naming convention.### Attributes| Name | Type | Description ||------|------|-------------|-| `import_statements` | `List[str]` | |-| `definitions` | `List[tuple[str, Any]]` | |+| `PAGE_DOWN` | `Any` | |+| `PAGE_UP` | `Any` | |+| `HOME` | `Any` | |+| `END` | `Any` | |+| `LEFT` | `Any` | |+| `RIGHT` | `Any` | |+| `UP` | `Any` | |+| `DOWN` | `Any` | |+| `RETURN` | `Any` | |+| `ENTER` | `Any` | |+| `ESCAPE` | `Any` | |+| `ESC` | `Any` | |+| `TAB` | `Any` | |+| `SPACE` | `Any` | |+| `BACKSPACE` | `Any` | |+| `DELETE` | `Any` | |+| `ALT` | `Any` | |+| `CTRL` | `Any` | |+| `SHIFT` | `Any` | |+| `WIN` | `Any` | |+| `COMMAND` | `Any` | |+| `OPTION` | `Any` | |+| `F1` | `Any` | |+| `F2` | `Any` | |+| `F3` | `Any` | |+| `F4` | `Any` | |+| `F5` | `Any` | |+| `F6` | `Any` | |+| `F7` | `Any` | |+| `F8` | `Any` | |+| `F9` | `Any` | |+| `F10` | `Any` | |+| `F11` | `Any` | |+| `F12` | `Any` | |-### set_default_computer+### Methods++#### Key.from_string```python-def set_default_computer(computer: Any) -> None+def from_string(cls, key: str) -> Key | str```-Set the default computer instance to be used by the remote decorator.+Convert a string key name to a Key enum value.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `computer` | `Any` | The computer instance to use as default |--### sandboxed--```python-def sandboxed(venv_name: str = 'default', computer: str = 'default', max_retries: int = 3) -> Callable[[Callable[P, R]], Callable[P, Awaitable[R]]]-```+| `key` | `Any` | String key name to convert |-Decorator that wraps a function to be executed remotely via computer.venv_exec+**Returns:** Key enum value if the string matches a known key, otherwise returns the original string for single character keys-The function is automatically analyzed for dependencies (imports, helper functions,-constants, etc.) and reconstructed with all necessary code in the remote sandbox.+----**Parameters:**+## DioramaComputer-| Name | Type | Description |-|------|------|-------------|-| `venv_name` | `Any` | Name of the virtual environment to execute in |-| `computer` | `Any` | The computer instance to use, or "default" to use the globally set default |-| `max_retries` | `Any` | Maximum number of retries for the remote execution |+A Computer-compatible proxy for Diorama that sends commands over the ComputerInterface.-### generate_source_code+### Constructor```python-def generate_source_code(func: FunctionType) -> str+DioramaComputer(self, computer, apps)```-Generate complete source code for a function with all dependencies.--**Parameters:**+### Attributes| Name | Type | Description ||------|------|-------------|-| `func` | `Any` | The function to generate source code for |+| `computer` | `Any` | |+| `apps` | `Any` | |+| `interface` | `Any` | |-**Returns:** Complete Python source code as a string+### Methods----+#### DioramaComputer.run-## models+```python+async def run(self)+```-Models for computer configuration.+Initialize and run the DioramaComputer if not already initialized.++**Returns:** self: The DioramaComputer instance----## BaseVMProvider+## DioramaComputerInterface-*Inherits from: AsyncContextManager*+Diorama Interface proxy that sends diorama_cmds via the Computer's interface.-Base interface for VM providers.+### Constructor-All VM provider implementations must implement this interface.+```python+DioramaComputerInterface(self, computer, apps)+```### Attributes| Name | Type | Description ||------|------|-------------|-| `provider_type` | `VMProviderType` | Get the provider type. |+| `computer` | `Any` | |+| `apps` | `Any` | |### Methods-#### BaseVMProvider.get_vm+#### DioramaComputerInterface.screenshot```python-async def get_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]+async def screenshot(self, as_bytes = True)```-Get VM information by name.+Take a screenshot of the diorama scene.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `name` | `Any` | Name of the VM to get information for |-| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |+| `as_bytes` | `bool` | If True, return image as bytes; if False, return PIL Image object |-**Returns:** Dictionary with VM information including status, IP address, etc.+**Returns:** bytes or PIL.Image: Screenshot data in the requested format-#### BaseVMProvider.list_vms+#### DioramaComputerInterface.get_screen_size```python-async def list_vms(self) -> ListVMsResponse+async def get_screen_size(self)```-List all available VMs.+Get the dimensions of the diorama scene.-**Returns:** ListVMsResponse: A list of minimal VM objects as defined in `computer.providers.types.MinimalVM`.+**Returns:** dict: Dictionary containing 'width' and 'height' keys with pixel dimensions-#### BaseVMProvider.run_vm+#### DioramaComputerInterface.move_cursor```python-async def run_vm(self, image: str, name: str, run_opts: Dict[str, Any], storage: Optional[str] = None) -> Dict[str, Any]+async def move_cursor(self, x, y)```-Run a VM by name with the given options.+Move the cursor to the specified coordinates.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `image` | `Any` | Name/tag of the image to use |-| `name` | `Any` | Name of the VM to run |-| `run_opts` | `Any` | Dictionary of run options (memory, cpu, etc.) |-| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |--**Returns:** Dictionary with VM run status and information+| `x` | `int` | X coordinate to move cursor to |+| `y` | `int` | Y coordinate to move cursor to |-#### BaseVMProvider.stop_vm+#### DioramaComputerInterface.left_click```python-async def stop_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]+async def left_click(self, x = None, y = None)```-Stop a VM by name.+Perform a left mouse click at the specified coordinates or current cursor position.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `name` | `Any` | Name of the VM to stop |-| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |--**Returns:** Dictionary with VM stop status and information+| `x` | `int, optional` | X coordinate to click at. If None, clicks at current cursor position |+| `y` | `int, optional` | Y coordinate to click at. If None, clicks at current cursor position |-#### BaseVMProvider.restart_vm+#### DioramaComputerInterface.right_click```python-async def restart_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]+async def right_click(self, x = None, y = None)```-Restart a VM by name.+Perform a right mouse click at the specified coordinates or current cursor position.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `name` | `Any` | Name of the VM to restart |-| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |--**Returns:** Dictionary with VM restart status and information+| `x` | `int, optional` | X coordinate to click at. If None, clicks at current cursor position |+| `y` | `int, optional` | Y coordinate to click at. If None, clicks at current cursor position |-#### BaseVMProvider.update_vm+#### DioramaComputerInterface.double_click```python-async def update_vm(self, name: str, update_opts: Dict[str, Any], storage: Optional[str] = None) -> Dict[str, Any]+async def double_click(self, x = None, y = None)```-Update VM configuration.+Perform a double mouse click at the specified coordinates or current cursor position.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `name` | `Any` | Name of the VM to update |-| `update_opts` | `Any` | Dictionary of update options (memory, cpu, etc.) |-| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |--**Returns:** Dictionary with VM update status and information+| `x` | `int, optional` | X coordinate to double-click at. If None, clicks at current cursor position |+| `y` | `int, optional` | Y coordinate to double-click at. If None, clicks at current cursor position |-#### BaseVMProvider.get_ip+#### DioramaComputerInterface.scroll_up```python-async def get_ip(self, name: str, storage: Optional[str] = None, retry_delay: int = 2) -> str+async def scroll_up(self, clicks = 1)```-Get the IP address of a VM, waiting indefinitely until it's available.+Scroll up by the specified number of clicks.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `name` | `Any` | Name of the VM to get the IP for |-| `storage` | `Any` | Optional storage path override. If provided, this will be used instead of the provider's default storage path. |-| `retry_delay` | `Any` | Delay between retries in seconds (default: 2) |--**Returns:** IP address of the VM when it becomes available-------## Display--Display configuration.+| `clicks` | `int` | Number of scroll clicks to perform upward. Defaults to 1 |-### Constructor+#### DioramaComputerInterface.scroll_down```python-Display(self, width: int, height: int) -> None+async def scroll_down(self, clicks = 1)```-### Attributes+Scroll down by the specified number of clicks.++**Parameters:**| Name | Type | Description ||------|------|-------------|-| `width` | `int` | |-| `height` | `int` | |-------## Image--VM image configuration.+| `clicks` | `int` | Number of scroll clicks to perform downward. Defaults to 1 |-### Constructor+#### DioramaComputerInterface.drag_to```python-Image(self, image: str, tag: str, name: str) -> None+async def drag_to(self, x, y, duration = 0.5)```-### Attributes+Drag from the current cursor position to the specified coordinates.++**Parameters:**| Name | Type | Description ||------|------|-------------|-| `image` | `str` | |-| `tag` | `str` | |-| `name` | `str` | |+| `x` | `int` | X coordinate to drag to |+| `y` | `int` | Y coordinate to drag to |+| `duration` | `float` | Duration of the drag operation in seconds. Defaults to 0.5 |----+#### DioramaComputerInterface.get_cursor_position-## Computer+```python+async def get_cursor_position(self)+```-Computer configuration.+Get the current cursor position.-### Constructor+**Returns:** dict: Dictionary containing the current cursor coordinates++#### DioramaComputerInterface.type_text```python-Computer(self, image: str, tag: str, name: str, display: Display, memory: str, cpu: str, vm_provider: Optional[BaseVMProvider] = None) -> None+async def type_text(self, text)```-### Attributes+Type the specified text at the current cursor position.++**Parameters:**| Name | Type | Description ||------|------|-------------|-| `image` | `str` | |-| `tag` | `str` | |-| `name` | `str` | |-| `display` | `Display` | |-| `memory` | `str` | |-| `cpu` | `str` | |-| `vm_provider` | `Optional[BaseVMProvider]` | |--### Methods+| `text` | `str` | The text to type |-#### Computer.get_ip+#### DioramaComputerInterface.press_key```python-async def get_ip(self) -> Optional[str]+async def press_key(self, key)```-Get the IP address of the VM.-----+Press a single key.-## tracing+**Parameters:**-Computer tracing functionality for recording sessions.+| Name | Type | Description |+|------|------|-------------|+| `key` | `Any` | The key to press |-This module provides a Computer.tracing API inspired by Playwright's tracing functionality,-allowing users to record computer interactions for debugging, training, and analysis.+#### DioramaComputerInterface.hotkey----+```python+async def hotkey(self, keys = ())+```-## ComputerTracing+Press multiple keys simultaneously as a hotkey combination.-Computer tracing class that records computer interactions and saves them to disk.+**Raises:**-This class provides a flexible API for recording computer sessions with configurable-options for what to record (screenshots, API calls, video, etc.).+- `ValueError` - If any key is not a Key enum or string type-### Constructor+#### DioramaComputerInterface.to_screen_coordinates```python-ComputerTracing(self, computer_instance)+async def to_screen_coordinates(self, x, y)```-### Attributes+Convert coordinates to screen coordinates.++**Parameters:**| Name | Type | Description ||------|------|-------------|-| `is_tracing` | `bool` | Check if tracing is currently active. |+| `x` | `int` | X coordinate to convert |+| `y` | `int` | Y coordinate to convert |-### Methods+**Returns:** dict: Dictionary containing the converted screen coordinates-#### ComputerTracing.start+----```python-async def start(self, config: Optional[Dict[str, Any]] = None) -> None-```+## helpers-Start tracing with the specified configuration.+Helper functions and decorators for the Computer module.-**Parameters:**+---++## DependencyInfo++*Inherits from: TypedDict*++### Attributes| Name | Type | Description ||------|------|-------------|-| `config` | `Any` | Tracing configuration dict with options: - video: bool - Record video frames (default: False) - screenshots: bool - Record screenshots (default: True) - api_calls: bool - Record API calls and results (default: True) - accessibility_tree: bool - Record accessibility tree snapshots (default: False) - metadata: bool - Record custom metadata (default: True) - name: str - Custom trace name (default: auto-generated) - path: str - Custom trace directory path (default: auto-generated) |+| `import_statements` | `List[str]` | |+| `definitions` | `List[tuple[str, Any]]` | |-#### ComputerTracing.stop+### set_default_computer```python-async def stop(self, options: Optional[Dict[str, Any]] = None) -> str+def set_default_computer(computer: Any) -> None```-Stop tracing and save the trace data.+Set the default computer instance to be used by the remote decorator.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `options` | `Any` | Stop options dict with: - path: str - Custom output path for the trace archive - format: str - Output format ('zip' or 'dir', default: 'zip') |--**Returns:** str: Path to the saved trace file or directory+| `computer` | `Any` | The computer instance to use as default |-#### ComputerTracing.record_api_call+### sandboxed```python-async def record_api_call(self, method: str, args: Dict[str, Any], result: Any = None, error: Optional[Exception] = None) -> None+def sandboxed(venv_name: str = 'default', computer: str = 'default', max_retries: int = 3) -> Callable[[Callable[P, R]], Callable[P, Awaitable[R]]]```-Record an API call event.+Decorator that wraps a function to be executed remotely via computer.venv_exec++The function is automatically analyzed for dependencies (imports, helper functions,+constants, etc.) and reconstructed with all necessary code in the remote sandbox.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `method` | `Any` | The method name that was called |-| `args` | `Any` | Arguments passed to the method |-| `result` | `Any` | Result returned by the method |-| `error` | `Any` | Exception raised by the method, if any |--#### ComputerTracing.record_accessibility_tree--```python-async def record_accessibility_tree(self) -> None-```--Record the current accessibility tree if enabled.+| `venv_name` | `Any` | Name of the virtual environment to execute in |+| `computer` | `Any` | The computer instance to use, or "default" to use the globally set default |+| `max_retries` | `Any` | Maximum number of retries for the remote execution |-#### ComputerTracing.add_metadata+### generate_source_code```python-async def add_metadata(self, key: str, value: Any) -> None+def generate_source_code(func: FunctionType) -> str```-Add custom metadata to the trace.+Generate complete source code for a function with all dependencies.**Parameters:**| Name | Type | Description ||------|------|-------------|-| `key` | `Any` | Metadata key |-| `value` | `Any` | Metadata value |+| `func` | `Any` | The function to generate source code for |++**Returns:** Complete Python source code as a string---diff --git a/docs/content/docs/cuabot/reference/index.mdx b/docs/content/docs/cuabot/reference/index.mdxindex 90eacfd324..abe9868fd9 100644--- a/docs/content/docs/cuabot/reference/index.mdx+++ b/docs/content/docs/cuabot/reference/index.mdx@@ -7,7 +7,7 @@ description: TypeScript API reference for the Cua-Bot sandboxed agent frameworkAUTO-GENERATED FILE - DO NOT EDIT DIRECTLYGenerated by: npx tsx scripts/docs-generators/typescript-sdk.tsSource: libs/cuabot/src- Version: 1.0.13+ Version: 1.0.14*/}import { Callout } from 'fumadocs-ui/components/callout';@@ -16,7 +16,7 @@ import { VersionHeader } from '@/components/version-selector';<VersionHeaderversions={[{"version":"1.0","href":"/cuabot/reference","isCurrent":true}]}currentVersion="1.0"- fullVersion="1.0.13"+ fullVersion="1.0.14"packageName="cuabot"installCommand="npm install -g cuabot"/>diff --git a/docs/content/docs/lume/reference/cli-reference.mdx b/docs/content/docs/lume/reference/cli-reference.mdxindex a8c3427314..1ee83e3451 100644--- a/docs/content/docs/lume/reference/cli-reference.mdx+++ b/docs/content/docs/lume/reference/cli-reference.mdx@@ -7,7 +7,7 @@ description: Command Line Interface reference for LumeAUTO-GENERATED FILE - DO NOT EDIT DIRECTLYGenerated by: npx tsx scripts/docs-generators/lume.tsSource: libs/lume/src/Commands/*.swift- Version: 0.2.76+ Version: 0.2.81*/}import { Callout } from 'fumadocs-ui/components/callout';@@ -16,7 +16,7 @@ import { VersionHeader } from '@/components/version-selector';<VersionHeaderversions={[{"version":"0.2","href":"/lume/reference/cli-reference","isCurrent":true}]}currentVersion="0.2"- fullVersion="0.2.76"+ fullVersion="0.2.81"packageName="lume"installCommand="curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh | bash"/>diff --git a/docs/content/docs/lume/reference/http-api.mdx b/docs/content/docs/lume/reference/http-api.mdxindex 524738783a..b205e76e9d 100644--- a/docs/content/docs/lume/reference/http-api.mdx+++ b/docs/content/docs/lume/reference/http-api.mdx@@ -7,7 +7,7 @@ description: HTTP API reference for Lume serverAUTO-GENERATED FILE - DO NOT EDIT DIRECTLYGenerated by: npx tsx scripts/docs-generators/lume.tsSource: libs/lume/src/Server/*.swift- Version: 0.2.76+ Version: 0.2.81*/}import { Callout } from 'fumadocs-ui/components/callout';@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';<VersionHeaderversions={[{"version":"0.2","href":"/lume/reference/cli-reference","isCurrent":true}]}currentVersion="0.2"- fullVersion="0.2.76"+ fullVersion="0.2.81"packageName="lume"installCommand="curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh | bash"/>@@ -392,6 +392,7 @@ Start a virtual machine| sharedDirectories | array | No | Directories to share with the VM || recoveryMode | boolean | No | Boot macOS VM in recovery mode (default: false) || storage | string | No | VM storage location |+| clipboard | boolean | No | Enable bidirectional clipboard sync via SSH (experimental) (default: false) |#### Example Requestdiff --git a/libs/python/agent/agent/decorators.py b/libs/python/agent/agent/decorators.pyindex 4c588a2660..c662221f38 100644--- a/libs/python/agent/agent/decorators.py+++ b/libs/python/agent/agent/decorators.py@@ -60,9 +60,34 @@ def get_agent_configs() -> List[AgentConfigInfo]:return _agent_configs.copy()+def _strip_cua_prefix(model: str) -> str:+ """Strip the ``cua/<provider>/`` routing prefix so the bare model name+ can be matched against registered agent patterns.++ Examples:+ cua/google/gemini-3-flash-preview -> gemini-3-flash-preview+ cua/anthropic/claude-sonnet-4-6 -> claude-sonnet-4-6+ gemini-3-flash-preview -> gemini-3-flash-preview (unchanged)+ """+ parts = model.split("/")+ if parts[0] == "cua" and len(parts) >= 3:+ return "/".join(parts[2:])+ return model++def find_agent_config(model: str) -> Optional[AgentConfigInfo]:- """Find the best matching agent config for a model"""+ """Find the best matching agent config for a model.++ For each registered config (checked in priority order), tries the+ original model string first and then the bare model name with the+ ``cua/<provider>/`` routing prefix stripped. This ensures that+ routed models (e.g. ``cua/google/gemini-3-flash-preview``) resolve+ to the same agent loop as their bare counterparts.+ """+ stripped = _strip_cua_prefix(model)for config_info in _agent_configs:if config_info.matches_model(model):return config_info+ if stripped != model and config_info.matches_model(stripped):+ return config_inforeturn Nonediff --git a/libs/python/agent/agent/loops/gemini.py b/libs/python/agent/agent/loops/gemini.pyindex e2411d8bfe..86221d779a 100644--- a/libs/python/agent/agent/loops/gemini.py+++ b/libs/python/agent/agent/loops/gemini.py@@ -27,6 +27,7 @@ from PIL import Imagefrom ..decorators import register_agentfrom ..loops.base import AsyncAgentConfig+from ..responses import make_reasoning_itemfrom ..types import AgentCapability@@ -92,6 +93,58 @@ def _sanitize_for_json(obj: Any) -> Any:return str(obj)+def _create_gemini_client(+ original_model: str, genai: Any, kwargs: Dict[str, Any]+) -> Tuple[Any, str]:+ """Create a Gemini SDK client, routing through CUA proxy if model has cua/ prefix.++ Returns (client, bare_model_name).++ When the model string starts with ``cua/<provider>/`` the Google GenAI SDK+ is configured to send requests through the CUA inference proxy at+ ``{CUA_BASE_URL}/gemini``. This keeps the Gemini loop as the single code+ path for both direct-Google and CUA-routed Gemini models.+ """+ import os++ from ..decorators import _strip_cua_prefix++ model = _strip_cua_prefix(original_model)+ is_cua_routed = original_model != model++ if is_cua_routed:+ api_key = (+ kwargs.get("api_key") or os.getenv("CUA_INFERENCE_API_KEY") or os.getenv("CUA_API_KEY")+ )+ if not api_key:+ raise ValueError(+ "No CUA API key found for cua/ model routing. "+ "Set CUA_API_KEY environment variable or pass api_key to ComputerAgent()."+ )+ cua_base_url = os.getenv("CUA_BASE_URL", "https://inference.cua.ai/v1")+ http_options: Dict[str, Any] = {"base_url": f"{cua_base_url}/gemini"}+ # Include CUA version headers if available+ try:+ from core.http import cua_version_headers++ hdrs = cua_version_headers()+ if hdrs:+ http_options["headers"] = hdrs+ except Exception:+ pass+ client = genai.Client(api_key=api_key, http_options=http_options)+ else:+ api_key = kwargs.get("api_key", os.getenv("GOOGLE_API_KEY"))+ if api_key:+ client = genai.Client(api_key=api_key)+ else:+ # Vertex AI mode - requires GOOGLE_CLOUD_PROJECT, GOOGLE_CLOUD_LOCATION env vars+ # and Application Default Credentials (ADC)+ client = genai.Client()++ return client, model++def _find_last_user_text(messages: List[Dict[str, Any]]) -> List[str]:texts: List[str] = []for msg in reversed(messages):@@ -219,7 +272,8 @@ def _convert_messages_to_gemini_contents(types.Content(role="user",parts=[- types.Part.from_bytes(data=img_bytes, mime_type="image/png")+ types.Part(text="[screenshot]"),+ types.Part.from_bytes(data=img_bytes, mime_type="image/png"),],))@@ -303,24 +357,36 @@ def _build_custom_function_declarations(types: Any) -> List[Any]:return [types.FunctionDeclaration(name="click_at",- description="Click at the specified x,y coordinates on the screen. Coordinates are normalized 0-999.",+ description="Click at the specified x,y coordinates on the screen. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen. Look carefully at the screenshot to identify the exact position of the target element before clicking.",parameters={"type": "object","properties": {- "x": {"type": "integer", "description": "X coordinate (0-999 normalized)"},- "y": {"type": "integer", "description": "Y coordinate (0-999 normalized)"},+ "x": {+ "type": "integer",+ "description": "X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",+ },+ "y": {+ "type": "integer",+ "description": "Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",+ },},"required": ["x", "y"],},),types.FunctionDeclaration(name="type_text_at",- description="Type text at the specified x,y coordinates. First clicks at the location, then types the text.",+ description="Type text at the specified x,y coordinates. First clicks at the location, then types the text. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen.",parameters={"type": "object","properties": {- "x": {"type": "integer", "description": "X coordinate (0-999 normalized)"},- "y": {"type": "integer", "description": "Y coordinate (0-999 normalized)"},+ "x": {+ "type": "integer",+ "description": "X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",+ },+ "y": {+ "type": "integer",+ "description": "Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",+ },"text": {"type": "string", "description": "Text to type"},"press_enter": {"type": "boolean",@@ -332,12 +398,18 @@ def _build_custom_function_declarations(types: Any) -> List[Any]:),types.FunctionDeclaration(name="hover_at",- description="Move the mouse cursor to the specified x,y coordinates without clicking.",+ description="Move the mouse cursor to the specified x,y coordinates without clicking. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen.",parameters={"type": "object","properties": {- "x": {"type": "integer", "description": "X coordinate (0-999 normalized)"},- "y": {"type": "integer", "description": "Y coordinate (0-999 normalized)"},+ "x": {+ "type": "integer",+ "description": "X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",+ },+ "y": {+ "type": "integer",+ "description": "Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",+ },},"required": ["x", "y"],},@@ -358,12 +430,18 @@ def _build_custom_function_declarations(types: Any) -> List[Any]:),types.FunctionDeclaration(name="scroll_at",- description="Scroll at the specified x,y coordinates in a given direction.",+ description="Scroll at the specified x,y coordinates in a given direction. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen.",parameters={"type": "object","properties": {- "x": {"type": "integer", "description": "X coordinate (0-999 normalized)"},- "y": {"type": "integer", "description": "Y coordinate (0-999 normalized)"},+ "x": {+ "type": "integer",+ "description": "X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",+ },+ "y": {+ "type": "integer",+ "description": "Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",+ },"direction": {"type": "string","enum": ["up", "down", "left", "right"],@@ -394,25 +472,25 @@ def _build_custom_function_declarations(types: Any) -> List[Any]:),types.FunctionDeclaration(name="drag_and_drop",- description="Drag from one coordinate to another.",+ description="Drag from one coordinate to another. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen.",parameters={"type": "object","properties": {"x": {"type": "integer",- "description": "Starting X coordinate (0-999 normalized)",+ "description": "Starting X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",},"y": {"type": "integer",- "description": "Starting Y coordinate (0-999 normalized)",+ "description": "Starting Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",},"destination_x": {"type": "integer",- "description": "Destination X coordinate (0-999 normalized)",+ "description": "Destination X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",},"destination_y": {"type": "integer",- "description": "Destination Y coordinate (0-999 normalized)",+ "description": "Destination Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",},},"required": ["x", "y", "destination_x", "destination_y"],@@ -625,19 +703,9 @@ class GeminiComputerUseConfig(AsyncAgentConfig):**kwargs,) -> Dict[str, Any]:genai, types = _lazy_import_genai()- import os- # Authentication follows two modes based on environment variables:- # 1. Google AI Studio: Set GOOGLE_API_KEY- # 2. Vertex AI: Set GOOGLE_CLOUD_PROJECT, GOOGLE_CLOUD_LOCATION, GOOGLE_GENAI_USE_VERTEXAI=True- api_key = kwargs.get("api_key", os.getenv("GOOGLE_API_KEY"))-- if api_key:- client = genai.Client(api_key=api_key)- else:- # Vertex AI mode - requires GOOGLE_CLOUD_PROJECT, GOOGLE_CLOUD_LOCATION env vars- # and Application Default Credentials (ADC)- client = genai.Client()+ # Create client with CUA routing support (detects cua/ prefix automatically)+ client, model = _create_gemini_client(model, genai, kwargs)# Extract Gemini 3-specific parameters# thinking_level: Use types.ThinkingLevel enum values (e.g., "LOW", "HIGH", "MEDIUM", "MINIMAL")@@ -678,6 +746,10 @@ class GeminiComputerUseConfig(AsyncAgentConfig):# Assume it's already an SDK enum valueresolved_media_resolution = media_resolution+ # Convert full message history to Gemini Contents format+ # (done early so screen dimensions are available for system instruction)+ contents, (screen_w, screen_h) = _convert_messages_to_gemini_contents(messages, types)+# Compose tools config based on model type# Gemini 2.5 Computer Use Preview uses built-in ComputerUse tool# Gemini 3 Flash/Pro Preview uses custom function declarations@@ -689,7 +761,19 @@ class GeminiComputerUseConfig(AsyncAgentConfig):print(f"[DEBUG] Using custom function declarations for Gemini 3 model: {model}")print(f"[DEBUG] Number of custom functions: {len(custom_functions)}")+ # Build system instruction with coordinate system and screen resolution context+ # to improve coordinate precision for Gemini 3 models+ system_instruction = (+ f"You are controlling a computer with screen resolution {screen_w}x{screen_h} pixels. "+ "When using coordinate-based functions (click_at, type_text_at, hover_at, scroll_at, drag_and_drop), "+ "provide x and y as normalized values in the 0-999 range: "+ "x=0 is the left edge, x=999 is the right edge; "+ "y=0 is the top edge, y=999 is the bottom edge. "+ "Look carefully at the screenshot to identify the exact position of UI elements before clicking."+ )+generate_content_config = types.GenerateContentConfig(+ system_instruction=system_instruction,tools=[types.Tool(function_declarations=custom_functions),],@@ -734,9 +818,6 @@ class GeminiComputerUseConfig(AsyncAgentConfig):media_resolution=resolved_media_resolution,)- # Convert full message history to Gemini Contents format- contents, (screen_w, screen_h) = _convert_messages_to_gemini_contents(messages, types)-api_kwargs = {"model": model,"contents": contents,@@ -800,6 +881,10 @@ class GeminiComputerUseConfig(AsyncAgentConfig):text_parts: List[str] = []function_calls: List[Dict[str, Any]] = []for p in candidate.content.parts:+ # Check for thinking/reasoning content first+ if getattr(p, "thought", False) and getattr(p, "text", None):+ output_items.append(make_reasoning_item(p.text))+ continueif getattr(p, "text", None):text_parts.append(p.text)if getattr(p, "function_call", None):@@ -844,14 +929,9 @@ class GeminiComputerUseConfig(AsyncAgentConfig):Returns pixel (x, y) if a click is proposed, else None."""genai, types = _lazy_import_genai()- import os- # Authentication: GOOGLE_API_KEY for AI Studio, or Vertex AI env vars- api_key = kwargs.get("api_key", os.getenv("GOOGLE_API_KEY"))- if api_key:- client = genai.Client(api_key=api_key)- else:- client = genai.Client()+ # Create client with CUA routing support (detects cua/ prefix automatically)+ client, model = _create_gemini_client(model, genai, kwargs)# Build tools config based on model typeis_gemini_3 = _is_gemini_3_model(model)@@ -860,12 +940,18 @@ class GeminiComputerUseConfig(AsyncAgentConfig):# For Gemini 3 models, use only click_at function declarationclick_function = types.FunctionDeclaration(name="click_at",- description="Click at the specified x,y coordinates on the screen. Coordinates are normalized 0-999.",+ description="Click at the specified x,y coordinates on the screen. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen. Look carefully at the screenshot to identify the exact position of the target element before clicking.",parameters={"type": "object","properties": {- "x": {"type": "integer", "description": "X coordinate (0-999 normalized)"},- "y": {"type": "integer", "description": "Y coordinate (0-999 normalized)"},+ "x": {+ "type": "integer",+ "description": "X coordinate (0-999 normalized). 0 is the left edge, 999 is the right edge.",+ },+ "y": {+ "type": "integer",+ "description": "Y coordinate (0-999 normalized). 0 is the top edge, 999 is the bottom edge.",+ },},"required": ["x", "y"],},diff --git a/uv.lock b/uv.lockindex 1abfbee7de..8064b7ce49 100644--- a/uv.lock+++ b/uv.lock@@ -1039,7 +1039,7 @@ wheels = [[[package]]name = "cua-agent"-version = "0.7.27"+version = "0.7.29"source = { editable = "libs/python/agent" }dependencies = [{ name = "aiohttp" },@@ -1189,6 +1189,33 @@ requires-dist = []provides-extras = ["openai", "anthropic", "qwen", "omni", "uitars", "uitars-mlx", "uitars-hf", "glm45v-hf", "opencua-hf", "internvl-hf", "moondream3", "ui", "cli", "hud", "gemini", "all"]+[[package]]+name = "cua-auto"+version = "0.1.1"+source = { editable = "libs/python/cua-auto" }+dependencies = [+ { name = "pillow" },+ { name = "pynput" },+ { name = "pyperclip" },+ { name = "pywinctl" },+]++[package.metadata]+requires-dist = [+ { name = "cua-auto", extras = ["mss", "windows", "pty"], marker = "extra == 'all'" },+ { name = "mss", marker = "extra == 'mss'", specifier = ">=9.0.0" },+ { name = "pillow", specifier = ">=10.0.0" },+ { name = "pynput", specifier = ">=1.7.0" },+ { name = "pyperclip", specifier = ">=1.9.0" },+ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" },+ { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" },+ { name = "pywin32", marker = "sys_platform == 'win32' and extra == 'windows'", specifier = ">=306" },+ { name = "pywinctl", specifier = ">=0.4" },+ { name = "pywinpty", marker = "sys_platform == 'win32' and extra == 'pty'", specifier = ">=2.0.0" },+ { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" },+]+provides-extras = ["mss", "windows", "pty", "all", "dev"]+[[package]]name = "cua-bench-ui"version = "0.7.0"@@ -1208,7 +1235,7 @@ requires-dist = [[[package]]name = "cua-computer"-version = "0.5.14"+version = "0.5.16"source = { editable = "libs/python/computer" }dependencies = [{ name = "aiohttp" },@@ -1252,10 +1279,11 @@ provides-extras = ["lume", "lumier", "ui", "all"][[package]]name = "cua-computer-server"-version = "0.3.18"+version = "0.3.19"source = { editable = "libs/python/computer-server" }dependencies = [{ name = "aiohttp" },+ { name = "cua-auto" },{ name = "cua-core" },{ name = "fastapi" },{ name = "fastmcp" },@@ -1291,6 +1319,7 @@ windows = [[package.metadata]requires-dist = [{ name = "aiohttp", specifier = ">=3.9.1" },+ { name = "cua-auto", editable = "libs/python/cua-auto" },{ name = "cua-core", editable = "libs/python/core" },{ name = "fastapi", specifier = ">=0.111.0" },{ name = "fastmcp", specifier = ">=2.0,<3" },@@ -1318,7 +1347,7 @@ provides-extras = ["macos", "linux", "windows"][[package]]name = "cua-core"-version = "0.1.17"+version = "0.1.18"source = { editable = "libs/python/core" }dependencies = [{ name = "posthog" },